Read about our upcoming Code of Conduct on this issue

This instance will be upgraded to Heptapod 0.28.1 on 2022-01-26 at 16:00 UTC+1 (a few minutes of down time)

Commit 7815edaf authored by Pierre Blanc-fatin's avatar Pierre Blanc-fatin
Browse files

Issue #14 Decorated methods can use locally imported functions

In this commit I have changed get_exterior_code() in analyse/__init__.py to resolve the double files writting problem .
I have also added a test for boosted methods that use a locally imported function.
And finally I have fixed the boosted functions name problem.

PS : An amend that consider Pierre Augier comments on the associated PR has been done.
parent 4de4a1da7c6f
import numpy as np
from transonic import boost
from exterior_import_boost import func_import
@boost
class MyClass2:
attr0: int
attr1: int
def __init__(self):
self.attr0 = self.attr1 = 1
@boost
def myfunc(self, arg: int):
return self.attr1 + self.attr0 + np.abs(arg) + func_import()
def check(self):
assert self.myfunc(1) == 5
......@@ -3,7 +3,6 @@ import numpy as np
from transonic import boost
from exterior_import_boost import func_import
# from transonic.exterior_import_jit import func_import
# transonic def func(
# float[][],
# float[][]
......
import h5py
import black
import numpy as np
from exterior_import_boost_2 import func_import_2
......@@ -11,5 +11,5 @@ def func_import():
return const + func_import_2() + np.pi - np.pi
def use_h5py():
print("use_h5py")
def use_black():
print(black)
import h5py
import black
import numpy as np
......@@ -10,5 +10,5 @@ def func_import_2():
return const
def use_h5py():
print("use_h5py")
def use_black():
print(black)
import numpy as np
from __MyClass2__exterior_import_boost import func_import
# pythran export __for_method__MyClass2__myfunc(int, int, int)
def __for_method__MyClass2__myfunc(self_attr0, self_attr1, arg):
return ((self_attr1 + self_attr0) + np.abs(arg)) + func_import()
# pythran export __code_new_method__MyClass2__myfunc
__code_new_method__MyClass2__myfunc = """
def new_method(self, arg):
return pythran_func(self.attr0, self.attr1, arg)
"""
# pythran export __transonic__
__transonic__ = ("0.2.3",)
import numpy as np
from __test__exterior_import_boost import func_import
from __func__exterior_import_boost import func_import
# pythran export func(float[][], float[][])
......
......@@ -24,7 +24,7 @@ deps =
git+https://github.com/zonca/pytest-ipynb#egg=pytest-ipynb
numpy
mpi4py
py{36,37}: black
black
pythran: pythran
cython: cython
......
......@@ -217,14 +217,13 @@ def analyse_aot(code, pathfile):
import sys
code_ext = {}
code_ext_cls = {}
code_ext = {"function": {}, "classe": {}}
def get_exterior_code(
codes_dependance: dict, previous_file_name=None, cls: str = None
codes_dependance: dict, previous_file_name=None, classes: str = None
):
""" Get all imported functions needed by jitted functions add multiple levels,
i.e get functions needed by functions needed by jitted function.
""" Get all imported functions needed by boosted functions and methods at multiple levels,
(i.e get functions needed by functions imported by boosted function) and add them into code_ext
"""
for func, dep in codes_dependance.items():
if dep:
......@@ -240,73 +239,42 @@ def analyse_aot(code, pathfile):
# get the content of the file
with open(str(file_path), "r") as file:
content = file.read()
file.close()
mod = extast.parse(content)
# filter the code and add it to code_ext dict
if cls:
code_ext_cls[file_name] = str(
filter_external_code(mod, node.names)
code_ext[classes][file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exterior codes
if code_ext[classes][file_name]:
get_exterior_code(
{func: code_ext[classes][file_name]},
file_name,
classes,
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
if not previous_file_name:
code_dependance = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext_cls[file_name]:
get_exterior_code(
{func: code_ext_cls[file_name]},
file_name,
cls="yes",
)
if previous_file_name:
code_ext_cls[
previous_file_name
] = change_import_name(
code_ext_cls[previous_file_name],
node,
func,
)
else:
code_ext[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
if not previous_file_name:
code_dependance = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext[file_name]:
get_exterior_code(
{func: code_ext[file_name]}, file_name
if previous_file_name:
code_ext[classes][
previous_file_name
] = change_import_name(
code_ext[classes][previous_file_name],
node,
func,
)
if previous_file_name:
code_ext[
previous_file_name
] = change_import_name(
code_ext[previous_file_name],
node,
func,
)
# TODO see if there is a cleaner way :
if "code_dependance" in locals():
return code_dependance
else:
return None
cod_dep = get_exterior_code({"test": code_dependance})
get_exterior_code({"test": code_dependance}, cls="yes")
if cod_dep:
code_dependance = cod_dep
return codes_dependance[func]
if boosted_dicts["functions"]:
func = next(iter(boosted_dicts["functions"]))
code_dependance = get_exterior_code(
{func: code_dependance}, classes="function"
)
if boosted_dicts["classes"]:
cls = next(iter(boosted_dicts["classes"]))
code_dependance = get_exterior_code(
{cls: code_dependance}, classes="classe"
)
debug(code_dependance)
debug("parse_code")
signatures_p = parse_code(code)
......@@ -328,12 +296,4 @@ def analyse_aot(code, pathfile):
annotations_sign.append(
{arg_name: type_ for arg_name, type_ in zip(arg_names, types)}
)
return (
boosted_dicts,
code_dependance,
annotations,
blocks,
code_ext,
code_ext_cls,
)
return boosted_dicts, code_dependance, annotations, blocks, code_ext
......@@ -106,7 +106,6 @@ def analysis_jit(code, pathfile):
# get the content of the file
with open(str(file_path), "r") as file:
content = file.read()
file.close()
mod = extast.parse(content)
if cls:
# filter the code and add it to code_ext dict
......@@ -114,8 +113,13 @@ def analysis_jit(code, pathfile):
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func, cls="yes"
codes_dependance_classes[
func
] = change_import_name(
codes_dependance_classes[func],
node,
func,
cls="yes",
)
# recursively get the exteriors codes
if code_ext_cls[file_name]:
......@@ -156,8 +160,9 @@ def analysis_jit(code, pathfile):
func,
)
if jitted_dicts["methods"]:
get_exterior_code(codes_dependance_classes, cls="yes")
get_exterior_code(codes_dependance)
get_exterior_code(codes_dependance_classes, cls="yes")
return (
jitted_dicts,
codes_dependance,
......
......@@ -174,6 +174,7 @@ def gather_rawcode_comments(node, code_module):
return rawcode, comments
packages_supported_by_pythran = ["numpy", "math", "functools", "cmath"]
# FIXME find path in non local imports
def find_path(node: object, pathfile: str):
""" Return the path of node (instance of ast.Import or ast.ImportFrom)
......@@ -183,7 +184,7 @@ def find_path(node: object, pathfile: str):
if isinstance(node, ast.ImportFrom):
name = node.module
if name in ["numpy", "math", "functools", "cmath"]:
if name in packages_supported_by_pythran:
return None, None
else:
parent = parent = Path(pathfile).parent
......@@ -191,7 +192,7 @@ def find_path(node: object, pathfile: str):
else:
# TODO complete the list
if node.names[0].name in ["numpy", "math", "functools", "cmath"]:
if node.names[0].name in packages_supported_by_pythran:
pass
else:
# TODO deal with an ast.Import
......@@ -222,7 +223,7 @@ def filter_external_code(module: object, names: list):
needed by functions or class in the parameter names
"""
code_dependance_annotations = ""
code = ""
lines_code = []
for node in module.body:
for name in names:
if isinstance(node, ast.FunctionDef):
......@@ -240,14 +241,14 @@ def filter_external_code(module: object, names: list):
usedef_chains=udc,
consider_annotations=None,
)
code += " \n " + str(extast.unparse(node))
lines_code.append(str(extast.unparse(node)))
code_dependance_annotations = capturex.make_code_external()
if isinstance(node, ast.Assign):
if node.targets[0].id == extast.unparse(name).rstrip("\n\r"):
code += str(extast.unparse(node))
lines_code.append(str(extast.unparse(node)))
if isinstance(node, ast.ClassDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
print_dumped(node)
code += str(extast.unparse(node))
lines_code.append(str(extast.unparse(node)))
return code_dependance_annotations + code
return code_dependance_annotations + "\n".join(lines_code)
......@@ -200,7 +200,7 @@ def make_pythran_code(path_py):
with open(path_py) as file:
code_module = file.read()
boosted_dicts, code_dependance, annotations, blocks, code_ext, code_ext_cls = analyse_aot(
boosted_dicts, code_dependance, annotations, blocks, code_ext = analyse_aot(
code_module, path_py
)
......@@ -299,4 +299,4 @@ def make_pythran_code(path_py):
code = format_str(code)
return code, code_ext, code_ext_cls
return code, code_ext
import h5py
import black
import numpy as np
from .for_test_exterior_import_jit_2 import func_import_2
......@@ -11,5 +11,5 @@ def func_import():
return const + np.pi - np.pi + func_import_2()
def use_h5py():
print("use_h5py")
def use_black():
print(black)
import h5py
import black
import numpy as np
......@@ -10,5 +10,5 @@ def func_import_2():
return const
def use_h5py():
print("use_h5py")
def use_black():
print(black)
......@@ -4,83 +4,83 @@ from transonic import jit, boost
from .mpi import Path
# def func0(a):
# return 2 * a
def func0(a):
return 2 * a
# def func():
# return 1
def func():
return 1
# func0_jitted = jit(func0)
func0_jitted = jit(func0)
# @jit
# def func1(a: "int[][] or float[]", l: "int list"):
# tmp = np.exp(sum(l))
# result = tmp * a * func0(a) + func()
# return result
@jit
def func1(a: "int[][] or float[]", l: "int list"):
tmp = np.exp(sum(l))
result = tmp * a * func0(a) + func()
return result
# @jit
# def func2(a):
# return a
@jit
def func2(a):
return a
# # weird but done on purpose for a better coverage
# Path(__file__).touch()
# weird but done on purpose for a better coverage
Path(__file__).touch()
# @jit()
# def func2(a):
# return a
@jit()
def func2(a):
return a
# @jit
# def func_dict(d: "str: float dict"):
# return d.popitem()
@jit
def func_dict(d: "str: float dict"):
return d.popitem()
# @jit
# def fib(n: int):
# """fibonacci"""
# if n < 2:
# return n
# return fib(n - 1) + fib(n - 2)
@jit
def fib(n: int):
"""fibonacci"""
if n < 2:
return n
return fib(n - 1) + fib(n - 2)
# @jit
# def use_fib():
# return [fib(n) for n in [1, 3, 5]]
@jit
def use_fib():
return [fib(n) for n in [1, 3, 5]]
# @boost
# class MyClass:
# def __init__(self):
# self.attr0 = self.attr1 = 1
@boost
class MyClass:
def __init__(self):
self.attr0 = self.attr1 = 1
# @jit()
# def myfunc(self, arg):
# return self.attr1 + self.attr0 + np.abs(arg)
@jit()
def myfunc(self, arg):
return self.attr1 + self.attr0 + np.abs(arg)
# @jit
# def myfunc1(self):
# return self.attr0
@jit
def myfunc1(self):
return self.attr0
# def check(self):
# assert self.myfunc(1) == 3
def check(self):
assert self.myfunc(1) == 3
from .for_test_exterior_import_jit import func_import, foo
# from numpy import pi
from numpy import pi
# const = 1
const = 1
# @jit
# def main(add: int):
# return foo + add + func_import() + pi - pi
@jit
def main(add: int):
return foo + add + func_import() + pi - pi
@boost
......@@ -93,4 +93,4 @@ class MyClass2:
return self.attr1 + self.attr0 + np.abs(arg) + func_import()
def check(self):
assert self.myfunc(1) == 4
assert self.myfunc(1) == 5
......@@ -74,6 +74,7 @@ from .util import (
import_from_path,
pythran,
is_method,
has_to_write,
)
from .annotation import make_signatures_from_typehinted_func, normalize_type_name
......@@ -128,18 +129,12 @@ class ModuleJIT:
# Write exterior code for functions
for file_name, code in self.code_ext.items():
path_ext = path_jit / self.module_name.replace(".", os.path.sep)
path_ext_file = path_ext / (file_name.replace(".", "/") + ".py")
path_ext_file = path_ext / (file_name + ".py")
path_ext_file.parent.mkdir(exist_ok=True, parents=True)
print(
"\033[33m",
"creation de ",
file_name,
" at ",
path_ext_file,
"\033[0m",
)
with open(path_ext_file, "w") as file:
file.write(code)
logger.info(f"{path_ext_file} written")
if has_to_write(path_ext_file, code):
with open(path_ext_file, "w") as file:
file.write(code)
# Write exterior code for classes
for file_name, code in self.code_ext_cls.items():
......@@ -148,18 +143,12 @@ class ModuleJIT:
/ "__jit_classes__"
/ self.module_name.replace(".", os.path.sep)
)
path_ext_file = path_ext / (file_name.replace(".", "/") + ".py")
path_ext_file = path_ext / (file_name + ".py")
path_ext_file.parent.mkdir(exist_ok=True, parents=True)
print(
"\033[33m",
"creation de ",
file_name,
" at ",
path_ext_file,
"\033[0m",
)
with open(path_ext_file, "w") as file:
file.write(code)
logger.info(f"{path_ext_file} written")
if has_to_write(path_ext_file, code):
with open(path_ext_file, "w") as file:
file.write(code)
def get_source(self):
if self.is_dummy_file:
......
......@@ -44,9 +44,8 @@ def test_detect_pythran_extensions():
make_backend_files((path_data_tests / name for name in names))
ext_names = detect_pythran_extensions(path_data_tests)
assert len(ext_names) == len(
names
) # -2 files (no_pythran.py and assign_fun_jit.py) +2 files (__test__exterior_import_boost.py and __test__exterior_import_boost_2.py)
# -2 files (no_pythran.py and assign_fun_jit.py) +2 files (__func__exterior_import_boost.py and __func__exterior_import_boost_2.py)
assert len(ext_names) == len(names)
shutil.rmtree(path_data_tests / "__pythran__", ignore_errors=True)
......
......@@ -50,98 +50,97 @@ if mpi.rank == 0:
mpi.barrier()
# def test_jit():
def test_jit():
from .for_test_justintime import func1
# from .for_test_justintime import func1
a = np.arange(2)
b = [1, 2]
# a = np.arange(2)
# b = [1, 2]
for _ in range(2):
func1(a, b)
sleep(0.1)
# for _ in range(2):
# func1(a, b)
# sleep(0.1)
def fib():
from .for_test_justintime import fib, use_fib
# def fib():
# from .for_test_justintime import fib, use_fib
# fib2 = fib(2)
# result = use_fib()
# wait_for_all_extensions()
# assert fib2 == fib(2)
# assert result == use_fib()
fib2 = fib(2)
result = use_fib()
wait_for_all_extensions()
assert fib2 == fib(2)
assert result == use_fib()