Read about our upcoming Code of Conduct on this issue

This instance will be upgraded to Heptapod 0.28.1 on 2022-01-26 at 16:00 UTC+1 (a few minutes of down time)

Commit 62bd0dcc authored by Pierre Blanc-fatin's avatar Pierre Blanc-fatin
Browse files

Report bug ModuleNotFound with a jitted method that use local imported functions

To have the error run 'make clean; pytest test_justintime.py -s'
I don't understand why function at line 277 in transonic.util.py (i.e "spec.loader.exec_module(module)") doesn't find the module imported by the module in its parameter"
I've written just before this line a test that looks for the existence of this module, and the test gives me a positive existence


In this commit I have only kept the test that give me this error (in test_justintime.py).
Be aware that this commit contains other unrelated changes like a factorisation of the function "filter_external_code"
parent 6cf408c69e81
......@@ -26,6 +26,7 @@ from .util import (
print_unparsed,
find_path,
change_import_name,
filter_external_code,
)
from .capturex import CaptureX
from .blocks_if import get_block_definitions
......@@ -216,44 +217,12 @@ def analyse_aot(code, pathfile):
import sys
def filter_external_code(module: object, names: list):
""" Filter the module to keep only the necessary nodes
needed by functions or class in the parameter names
"""
import gast as ast
code_dependance_annotations = ""
code = ""
for node in module.body:
for name in names:
if isinstance(node, ast.FunctionDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
ancestors, duc, udc = compute_ancestors_chains(module)
capturex = CaptureX(
[node],
module,
ancestors,
defuse_chains=duc,
usedef_chains=udc,
consider_annotations=None,
)
code += " \n " + str(extast.unparse(node))
code_dependance_annotations = (
capturex.make_code_external()
)
if isinstance(node, ast.Assign):
if node.targets[0].id == extast.unparse(name).rstrip("\n\r"):
code += str(extast.unparse(node))
if isinstance(node, ast.ClassDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
print_dumped(node)
code += str(extast.unparse(node))
return code_dependance_annotations + code
code_ext = {}
code_ext_cls = {}
def get_exterior_code(codes_dependance: dict, previous_file_name=None):
def get_exterior_code(
codes_dependance: dict, previous_file_name=None, cls: str = None
):
""" Get all imported functions needed by jitted functions add multiple levels,
i.e get functions needed by functions needed by jitted function.
"""
......@@ -274,29 +243,60 @@ def analyse_aot(code, pathfile):
file.close()
mod = extast.parse(content)
# filter the code and add it to code_ext dict
code_ext[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
if not previous_file_name:
code_dependance = change_import_name(
if cls:
code_ext_cls[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext[file_name]:
get_exterior_code(
{func: code_ext[file_name]}, file_name
if not previous_file_name:
code_dependance = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext_cls[file_name]:
get_exterior_code(
{func: code_ext_cls[file_name]},
file_name,
cls="yes",
)
if previous_file_name:
code_ext_cls[
previous_file_name
] = change_import_name(
code_ext_cls[previous_file_name],
node,
func,
)
else:
code_ext[file_name] = str(
filter_external_code(mod, node.names)
)
if previous_file_name:
code_ext[
previous_file_name
] = change_import_name(
code_ext[previous_file_name], node, func
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
if not previous_file_name:
code_dependance = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext[file_name]:
get_exterior_code(
{func: code_ext[file_name]}, file_name
)
if previous_file_name:
code_ext[
previous_file_name
] = change_import_name(
code_ext[previous_file_name],
node,
func,
)
# TODO see if there is a cleaner way :
if "code_dependance" in locals():
return code_dependance
......@@ -304,6 +304,7 @@ def analyse_aot(code, pathfile):
return None
cod_dep = get_exterior_code({"test": code_dependance})
get_exterior_code({"test": code_dependance}, cls="yes")
if cod_dep:
code_dependance = cod_dep
debug(code_dependance)
......@@ -328,4 +329,11 @@ def analyse_aot(code, pathfile):
{arg_name: type_ for arg_name, type_ in zip(arg_names, types)}
)
return boosted_dicts, code_dependance, annotations, blocks, code_ext
return (
boosted_dicts,
code_dependance,
annotations,
blocks,
code_ext,
code_ext_cls,
)
......@@ -11,7 +11,7 @@ from transonic.analyses import compute_ancestors_chains, get_decorated_dicts
from transonic.analyses.capturex import CaptureX
from transonic.analyses.util import print_dumped
from .util import find_path, change_import_name
from .util import find_path, change_import_name, filter_external_code
def analysis_jit(code, pathfile):
......@@ -83,42 +83,12 @@ def analysis_jit(code, pathfile):
import sys
def filter_external_code(module: object, names: list):
""" Filter the module to keep only the necessary nodes
needed by functions or class in the parameter names
"""
code_dependance_annotations = ""
code = ""
for node in module.body:
for name in names:
if isinstance(node, ast.FunctionDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
ancestors, duc, udc = compute_ancestors_chains(module)
capturex = CaptureX(
[node],
module,
ancestors,
defuse_chains=duc,
usedef_chains=udc,
consider_annotations=None,
)
code += " \n " + str(extast.unparse(node))
code_dependance_annotations = (
capturex.make_code_external()
)
if isinstance(node, ast.Assign):
if node.targets[0].id == extast.unparse(name).rstrip("\n\r"):
code += str(extast.unparse(node))
if isinstance(node, ast.ClassDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
print_dumped(node)
code += str(extast.unparse(node))
return code_dependance_annotations + code
code_ext = {}
code_ext_cls = {}
def get_exterior_code(codes_dependance: dict, previous_file_name=None):
def get_exterior_code(
codes_dependance: dict, previous_file_name=None, cls=None
):
""" Get all imported functions needed by jitted functions add multiple levels,
i.e get functions needed by functions needed by jitted function.
"""
......@@ -138,25 +108,60 @@ def analysis_jit(code, pathfile):
content = file.read()
file.close()
mod = extast.parse(content)
# filter the code and add it to code_ext dict
code_ext[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
# recursively get the exteriors codes
if code_ext[file_name]:
get_exterior_code(
{func: code_ext[file_name]}, file_name
if cls:
# filter the code and add it to code_ext dict
code_ext_cls[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func, cls="yes"
)
# recursively get the exteriors codes
if code_ext_cls[file_name]:
get_exterior_code(
{func: code_ext_cls[file_name]},
file_name,
cls="yes",
)
if previous_file_name:
code_ext_cls[
previous_file_name
] = change_import_name(
code_ext_cls[previous_file_name],
node,
func,
cls="yes",
)
else:
# filter the code and add it to code_ext dict
code_ext[file_name] = str(
filter_external_code(mod, node.names)
)
# change imported module names
codes_dependance[func] = change_import_name(
codes_dependance[func], node, func
)
if previous_file_name:
code_ext[
previous_file_name
] = change_import_name(
code_ext[previous_file_name], node, func
# recursively get the exteriors codes
if code_ext[file_name]:
get_exterior_code(
{func: code_ext[file_name]}, file_name
)
if previous_file_name:
code_ext[
previous_file_name
] = change_import_name(
code_ext[previous_file_name],
node,
func,
)
get_exterior_code(codes_dependance)
return (jitted_dicts, codes_dependance, codes_dependance_classes, code_ext)
get_exterior_code(codes_dependance_classes, cls="yes")
return (
jitted_dicts,
codes_dependance,
codes_dependance_classes,
code_ext,
code_ext_cls,
)
......@@ -2,10 +2,12 @@
=============================
"""
import beniget
from pathlib import Path
from textwrap import dedent
import gast as ast
from transonic.analyses import extast
from transonic.analyses.capturex import CaptureX
import astunparse
......@@ -176,8 +178,6 @@ def gather_rawcode_comments(node, code_module):
def find_path(node: object, pathfile: str):
""" Return the path of node (instance of ast.Import or ast.ImportFrom)
"""
import gast as ast
name = str()
path = str()
......@@ -199,7 +199,9 @@ def find_path(node: object, pathfile: str):
return name, path
def change_import_name(code_dep: str, changed_node: object, func_name: str):
def change_import_name(
code_dep: str, changed_node: object, func_name: str, cls: str = None
):
""" Change the name of changed_node in code_dep by adding "__"+func+"__"
at the beginning of the imported module, and return the modified code
"""
......@@ -210,4 +212,42 @@ def change_import_name(code_dep: str, changed_node: object, func_name: str):
node.module = "__" + func_name + "__" + node.module
elif isinstance(node, ast.Import):
node.names[0].name = "__" + func_name + "__" + node.names[0].name
if cls:
node.level = 0
return extast.unparse(mod)
def filter_external_code(module: object, names: list):
""" Filter the module to keep only the necessary nodes
needed by functions or class in the parameter names
"""
code_dependance_annotations = ""
code = ""
for node in module.body:
for name in names:
if isinstance(node, ast.FunctionDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
ancestors = beniget.Ancestors()
ancestors.visit(module)
duc = beniget.DefUseChains()
duc.visit(module)
udc = beniget.UseDefChains(duc)
capturex = CaptureX(
[node],
module,
ancestors,
defuse_chains=duc,
usedef_chains=udc,
consider_annotations=None,
)
code += " \n " + str(extast.unparse(node))
code_dependance_annotations = capturex.make_code_external()
if isinstance(node, ast.Assign):
if node.targets[0].id == extast.unparse(name).rstrip("\n\r"):
code += str(extast.unparse(node))
if isinstance(node, ast.ClassDef):
if node.name == extast.unparse(name).rstrip("\n\r"):
print_dumped(node)
code += str(extast.unparse(node))
return code_dependance_annotations + code
......@@ -200,7 +200,7 @@ def make_pythran_code(path_py):
with open(path_py) as file:
code_module = file.read()
boosted_dicts, code_dependance, annotations, blocks, code_ext = analyse_aot(
boosted_dicts, code_dependance, annotations, blocks, code_ext, code_ext_cls = analyse_aot(
code_module, path_py
)
......@@ -299,4 +299,4 @@ def make_pythran_code(path_py):
code = format_str(code)
return code, code_ext
return code, code_ext, code_ext_cls
......@@ -8,7 +8,7 @@ foo = 1
def func_import():
return const + func_import_2() + np.pi - np.pi
return const + np.pi - np.pi + func_import_2()
def use_h5py():
......
......@@ -4,79 +4,93 @@ from transonic import jit, boost
from .mpi import Path
def func0(a):
return 2 * a
# def func0(a):
# return 2 * a
def func():
return 1
# def func():
# return 1
func0_jitted = jit(func0)
# func0_jitted = jit(func0)
@jit
def func1(a: "int[][] or float[]", l: "int list"):
tmp = np.exp(sum(l))
result = tmp * a * func0(a) + func()
return result
# @jit
# def func1(a: "int[][] or float[]", l: "int list"):
# tmp = np.exp(sum(l))
# result = tmp * a * func0(a) + func()
# return result
@jit
def func2(a):
return a
# @jit
# def func2(a):
# return a
# weird but done on purpose for a better coverage
Path(__file__).touch()
# # weird but done on purpose for a better coverage
# Path(__file__).touch()
@jit()
def func2(a):
return a
# @jit()
# def func2(a):
# return a
@jit
def func_dict(d: "str: float dict"):
return d.popitem()
# @jit
# def func_dict(d: "str: float dict"):
# return d.popitem()
@jit
def fib(n: int):
"""fibonacci"""
if n < 2:
return n
return fib(n - 1) + fib(n - 2)
# @jit
# def fib(n: int):
# """fibonacci"""
# if n < 2:
# return n
# return fib(n - 1) + fib(n - 2)
@jit
def use_fib():
return [fib(n) for n in [1, 3, 5]]
# @jit
# def use_fib():
# return [fib(n) for n in [1, 3, 5]]
@boost
class MyClass:
def __init__(self):
self.attr0 = self.attr1 = 1
# @boost
# class MyClass:
# def __init__(self):
# self.attr0 = self.attr1 = 1
@jit()
def myfunc(self, arg):
return self.attr1 + self.attr0 + np.abs(arg)
# @jit()
# def myfunc(self, arg):
# return self.attr1 + self.attr0 + np.abs(arg)
@jit
def myfunc1(self):
return self.attr0
# @jit
# def myfunc1(self):
# return self.attr0
def check(self):
assert self.myfunc(1) == 3
# def check(self):
# assert self.myfunc(1) == 3
from .for_test_exterior_import_jit import func_import, foo
from numpy import pi
const = 1
# from numpy import pi
# const = 1
# @jit
# def main(add: int):
# return foo + add + func_import() + pi - pi
@jit
def main(add: int):
return foo + add + func_import() + pi - pi
@boost
class MyClass2:
def __init__(self):
self.attr0 = self.attr1 = 1
@jit()
def myfunc(self, arg):
return self.attr1 + self.attr0 + np.abs(arg) + func_import()
def check(self):
assert self.myfunc(1) == 4
......@@ -122,14 +122,42 @@ class ModuleJIT:
self.jit_functions = {}
source = self.get_source()
self.jitted_dicts, self.codes_dependance, self.codes_dependance_classes, self.code_ext = analysis_jit(
self.jitted_dicts, self.codes_dependance, self.codes_dependance_classes, self.code_ext, self.code_ext_cls = analysis_jit(
source, self.filename
)
# Write exterior code for functions
for file_name, code in self.code_ext.items():
path_ext = path_jit / self.module_name.replace(".", os.path.sep)
path_ext_file = path_ext / (file_name.replace(".", "/") + ".py")
path_ext_file.parent.mkdir(exist_ok=True, parents=True)
print(
"\033[33m",
"creation de ",
file_name,
" at ",
path_ext_file,
"\033[0m",
)
with open(path_ext_file, "w") as file:
file.write(code)
# Write exterior code for classes
for file_name, code in self.code_ext_cls.items():
path_ext = (
path_jit.parent
/ "__jit_classes__"
/ self.module_name.replace(".", os.path.sep)
)
path_ext_file = path_ext / (file_name.replace(".", "/") + ".py")
path_ext_file.parent.mkdir(exist_ok=True, parents=True)
print(
"\033[33m",
"creation de ",
file_name,
" at ",
path_ext_file,
"\033[0m",
)
with open(path_ext_file, "w") as file:
file.write(code)
......
......@@ -50,93 +50,110 @@ if mpi.rank == 0:
mpi.barrier()
def test_jit():
# def test_jit():
from .for_test_justintime import func1
# from .for_test_justintime import func1
a = np.arange(2)
b = [1, 2]
# a = np.arange(2)
# b = [1, 2]
for _ in range(2):
func1(a, b)
sleep(0.1)
# for _ in range(2):
# func1(a, b)
# sleep(0.1)
def fib():
from .for_test_justintime import fib, use_fib
# def fib():
# from .for_test_justintime import fib, use_fib
fib2 = fib(2)
result = use_fib()
wait_for_all_extensions()
assert fib2 == fib(2)
assert result == use_fib()