Read about our upcoming Code of Conduct on this issue

This instance will be upgraded to Heptapod 0.28.1 on 2022-01-26 at 16:00 UTC+1 (a few minutes of down time)

Commit 43332665 authored by Pierre Blanc-fatin's avatar Pierre Blanc-fatin
Browse files

Issue #14 imports supported by boosted functions + reorganisation + tests

Of course boosted functions support multi-levels of import as jitted function.
The reorganisation was done for factoring 2 identical functions in analyse_aot() and analyse_jit().
There is a new test about multi-levels import in boosted function.
parent e7d2e868c7d3
import numpy as np
from transonic import boost
from exterior_import_boost import func_import
# from transonic.exterior_import_jit import func_import
# transonic def func(
# float[][],
# float[][]
# )
# transonic def func(int[][], float[][])
@boost
def func(a, b):
return (a * np.log(b)).max() + func_import()
import h5py
import numpy as np
from .exterior_import_jit_2 import func_import_2
from exterior_import_boost_2 import func_import_2
const = 1
......
import numpy as np
from __test__exterior_import_boost import func_import
# pythran export func(float[][], float[][])
# pythran export func(int[][], float[][])
def func(a, b):
return (a * np.log(b)).max() + func_import()
# pythran export __transonic__
__transonic__ = ("0.2.3",)
......@@ -17,7 +17,6 @@ from pprint import pformat
import gast as ast
import beniget
from transonic.log import logger
from .util import (
......@@ -25,6 +24,8 @@ from .util import (
get_annotations,
print_dumped,
print_unparsed,
find_path,
change_import_name,
)
from .capturex import CaptureX
from .blocks_if import get_block_definitions
......@@ -112,7 +113,7 @@ def get_decorated_dicts(module, ancestors, duc, decorator="boost"):
return decorated_dicts
def analyse_aot(code):
def analyse_aot(code, pathfile):
"""Gather the informations for ``@boost`` and blocks"""
debug = logger.debug
......@@ -212,6 +213,99 @@ def analyse_aot(code):
)
code_dependance = capturex.make_code_external()
import sys
def filter_external_code(module: object, names: list):
""" Filter the module to keep only the necessary nodes
needed by functions or class in the parameter names
"""
import gast as ast
code_dependance_annotations = ""
code = ""
for node in module.body:
for name in names:
if isinstance(node, ast.FunctionDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
ancestors, duc, udc = compute_ancestors_chains(module)
capturex = CaptureX(
[node],
module,
ancestors,
defuse_chains=duc,
usedef_chains=udc,
consider_annotations=None,
)
code += " \n " + str(extast.unparse(node))
code_dependance_annotations = (
capturex.make_code_external()
)
if isinstance(node, ast.Assign):
if node.targets[0].id == extast.unparse(name).rstrip("\n\r"):
code += str(extast.unparse(node))
if isinstance(node, ast.ClassDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
print_dumped(node)
code += str(extast.unparse(node))
return code_dependance_annotations + code
code_ext = {}
def get_exterior_code(codes_dependance: dict, previous_file_name=None):
""" Get all imported functions needed by jitted functions add multiple levels,
i.e get functions needed by functions needed by jitted function.
"""
for func, dep in codes_dependance.items():
if dep:
module_ext = extast.parse(dep)
for node in module_ext.body:
if isinstance(node, ast.ImportFrom) or isinstance(
node, ast.Import
):
# get the path of the imported module
file_name, file_path = find_path(node, pathfile)
if file_name:
file_name = "__" + func + "__" + file_name
# get the content of the file
with open(str(file_path), "r") as file:
content = file.read()
file.close()
mod = extast.parse(content)
# filter the code and add it to code_ext dict
code_ext[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
if not previous_file_name:
code_dependance = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext[file_name]:
get_exterior_code(
{func: code_ext[file_name]}, file_name
)
if previous_file_name:
code_ext[
previous_file_name
] = change_import_name(
code_ext[previous_file_name], node, func
)
# TODO see if there is a cleaner way :
if "code_dependance" in locals():
return code_dependance
else:
return None
cod_dep = get_exterior_code({"test": code_dependance})
if cod_dep:
code_dependance = cod_dep
debug(code_dependance)
debug("parse_code")
signatures_p = parse_code(code)
......@@ -234,4 +328,4 @@ def analyse_aot(code):
{arg_name: type_ for arg_name, type_ in zip(arg_names, types)}
)
return boosted_dicts, code_dependance, annotations, blocks
return boosted_dicts, code_dependance, annotations, blocks, code_ext
......@@ -3,7 +3,6 @@
"""
import gast as ast
from path import Path
from transonic.log import logger
......@@ -12,6 +11,7 @@ from transonic.analyses import compute_ancestors_chains, get_decorated_dicts
from transonic.analyses.capturex import CaptureX
from transonic.analyses.util import print_dumped
from .util import find_path, change_import_name
def analysis_jit(code, pathfile):
......@@ -87,8 +87,6 @@ def analysis_jit(code, pathfile):
""" Filter the module to keep only the necessary nodes
needed by functions or class in the parameter names
"""
import gast as ast
code_dependance_annotations = ""
code = ""
for node in module.body:
......@@ -118,42 +116,6 @@ def analysis_jit(code, pathfile):
return code_dependance_annotations + code
# FIXME find path in non local imports
def find_path(node: object):
""" Return the path of node in parameter (instance of ast.Import or ast.ImportFrom)
"""
name = str()
path = str()
if isinstance(node, ast.ImportFrom):
name = node.module
if name in ["numpy", "math", "functools", "cmath"]:
return None, None
else:
parent = Path(pathfile).parent
path = parent / (name.replace(".", "/")) + ".py"
else:
# TODO complete the list
if node.names[0].name in ["numpy", "math", "functools", "cmath"]:
pass
else:
# TODO deal with an ast.Import
raise NotImplementedError
return name, path
def change_import_name(code_dep: str, changed_node: object, func: str):
""" Change the name of changed_node in code_dep by adding "__"+func+"__"
at the beginning of the imported module, and return the modified code
"""
mod = extast.parse(code_dep)
for node in mod.body:
if extast.unparse(node) == extast.unparse(changed_node):
if isinstance(node, ast.ImportFrom):
node.module = "__" + func + "__" + node.module
elif isinstance(node, ast.Import):
node.names[0].name = "__" + func + "__" + node.names[0].name
return extast.unparse(mod)
code_ext = {}
def get_exterior_code(codes_dependance: dict, previous_file_name=None):
......@@ -168,7 +130,7 @@ def analysis_jit(code, pathfile):
node, ast.Import
):
# get the path of the imported module
file_name, file_path = find_path(node)
file_name, file_path = find_path(node, pathfile)
if file_name:
file_name = "__" + func + "__" + file_name
# get the content of the file
......
......@@ -2,9 +2,8 @@
=============================
"""
from pathlib import Path
from textwrap import dedent
import gast as ast
from transonic.analyses import extast
import astunparse
......@@ -171,3 +170,44 @@ def gather_rawcode_comments(node, code_module):
)
)
return rawcode, comments
# FIXME find path in non local imports
def find_path(node: object, pathfile: str):
""" Return the path of node (instance of ast.Import or ast.ImportFrom)
"""
import gast as ast
name = str()
path = str()
if isinstance(node, ast.ImportFrom):
name = node.module
if name in ["numpy", "math", "functools", "cmath"]:
return None, None
else:
parent = parent = Path(pathfile).parent
path = parent / Path(str(name.replace(".", "/")) + ".py")
else:
# TODO complete the list
if node.names[0].name in ["numpy", "math", "functools", "cmath"]:
pass
else:
# TODO deal with an ast.Import
raise NotImplementedError
return name, path
def change_import_name(code_dep: str, changed_node: object, func_name: str):
""" Change the name of changed_node in code_dep by adding "__"+func+"__"
at the beginning of the imported module, and return the modified code
"""
mod = extast.parse(code_dep)
for node in mod.body:
if extast.unparse(node) == extast.unparse(changed_node):
if isinstance(node, ast.ImportFrom):
node.module = "__" + func_name + "__" + node.module
elif isinstance(node, ast.Import):
node.names[0].name = "__" + func_name + "__" + node.names[0].name
return extast.unparse(mod)
......@@ -200,7 +200,9 @@ def make_pythran_code(path_py):
with open(path_py) as file:
code_module = file.read()
boosted_dicts, code_dependance, annotations, blocks = analyse_aot(code_module)
boosted_dicts, code_dependance, annotations, blocks, code_ext = analyse_aot(
code_module, path_py
)
code = ["\n" + code_dependance + "\n"]
......@@ -297,4 +299,4 @@ def make_pythran_code(path_py):
code = format_str(code)
return code
return code, code_ext
import h5py
import numpy as np
from .for_test_exterior_import_jit_2 import func_import_2
const = 1
foo = 1
def func_import():
return const + func_import_2() + np.pi - np.pi
def use_h5py():
print("use_h5py")
import h5py
import numpy as np
const = 1
foo = 1
def func_import_2():
return const
def use_h5py():
print("use_h5py")
......@@ -71,7 +71,7 @@ class MyClass:
assert self.myfunc(1) == 3
from .exterior_import_jit import func_import, foo
from .for_test_exterior_import_jit import func_import, foo
from numpy import pi
const = 1
......
......@@ -39,11 +39,14 @@ def test_detect_pythran_extensions():
"mixed_classic_type_hint.py",
"assign_func_boost.py",
"assign_func_jit.py",
"boosted_func_use_import.py",
]
make_backend_files((path_data_tests / name for name in names))
ext_names = detect_pythran_extensions(path_data_tests)
assert len(ext_names) == len(names) - 2
assert len(ext_names) == len(
names
) # -2 files (no_pythran.py and assign_fun_jit.py) +2 files (__test__exterior_import_boost.py and __test__exterior_import_boost_2.py)
shutil.rmtree(path_data_tests / "__pythran__", ignore_errors=True)
......
......@@ -31,7 +31,13 @@ def test_create_pythran_files():
run()
# At this point, we can compare the produced files with saved files.
no_compare = ["no_pythran_.py", "assign_func_jit.py"]
# For exterior files, we can't compare cause transonic changes their names
no_compare = [
"no_pythran_.py",
"assign_func_jit.py",
"exterior_import_boost.py",
"exterior_import_boost_2.py",
]
for path in paths:
if path.name in no_compare:
continue
......
......@@ -232,7 +232,14 @@ def make_pythran_file(path_py: Path, force=False, log_level=None):
logger.warning(f"File {path_pythran} already up-to-date.")
return
code_pythran = make_pythran_code(path_py)
code_pythran, code_ext = make_pythran_code(path_py)
for file_name, code in code_ext.items():
path_ext = path_dir
path_ext_file = path_ext / (file_name.replace(".", "/") + ".py")
path_ext_file.parent.mkdir(exist_ok=True, parents=True)
with open(path_ext_file, "w") as file:
file.write(code)
if not code_pythran:
return
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment