Commit 39154112 authored by Pierre Augier's avatar Pierre Augier
Browse files

Fix cachedjit when calling with new types

parent 6a377bbcfc50
......@@ -18,6 +18,6 @@ dist*
*.egg-info/*
**/_pythran/_*
**/__pythran__/_*
fluidpythran/_path_data_tests.py
\ No newline at end of file
0.0.8 (?)
---------
- Fix cachedjit when calling with new types
0.0.7 (2018-11-15)
------------------
......
......@@ -17,7 +17,9 @@ if __name__ == "__main__":
a = b = np.zeros([2, 3])
for _ in range(40):
print(_)
for i in range(40):
print(f"{i}, call with arrays", end=", ")
func1(a, b)
print("call with numbers")
func1(1, 1.)
sleep(1)
......@@ -153,9 +153,9 @@ class FluidPythran:
if "." in module_name:
package, module = module_name.rsplit(".", 1)
module_pythran = package + "._pythran._" + module
module_pythran = package + ".__pythran__._" + module
else:
module_pythran = "_pythran._" + module_name
module_pythran = "__pythran__._" + module_name
try:
self.module_pythran = importlib.import_module(module_pythran)
......
......@@ -32,6 +32,8 @@ Internal API
:members:
:private-members:
.. autofunction:: import_from_path
Notes
-----
......@@ -63,11 +65,15 @@ used.
import inspect
import itertools
from pathlib import Path
import os
import sys
import importlib
import subprocess
import multiprocessing
import time
import sysconfig
import importlib.util
import hashlib
try:
import pythran
......@@ -79,10 +85,15 @@ from .annotation import make_signatures_from_typehinted_func
modules = {}
ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
if pythran and pythran.__version__ <= "0.9":
# avoid a Pythran bug with -o option
# it is bad because then we do not support using many Python versions
ext_suffix = "." + ext_suffix.rsplit(".", 1)[-1]
path_root = Path.home() / ".fluidpythran"
# weird name to avoid name collision
path_cachedjit = path_root / "_fp_cachedjit"
path_cachedjit = path_root / "__cachedjit__"
path_cachedjit.mkdir(parents=True, exist_ok=True)
......@@ -199,26 +210,19 @@ def cachedjit(func=None, native=True, xsimd=True, openmp=False):
return decor
def reimport_module(name_mod, avoid_extension=True):
sys.path.insert(0, str(path_root))
if name_mod in sys.modules:
mod = sys.modules[name_mod]
if avoid_extension and hasattr(mod, "__pythran__"):
raise NotImplementedError(
"It seems complicated and hacky to reload a C-extension! "
"You should be able to use type hints to avoid this bug..."
)
# If we don't do that, we get a segfault!
# see https://stackoverflow.com/questions/8295555/how-to-reload-a-python3-c-extension-module
# https://gist.github.com/TheWaWaR/d3c630f72dd631a0f336
def import_from_path(path, module_name):
"""Import a .py file or an extension from its path
del sys.modules[name_mod]
"""
package_name, mod_name = module_name.rsplit(".", 1)
name_file = path.name.split(".", 1)[0]
if mod_name != name_file:
module_name = ".".join((package_name, name_file))
print(f"(re)import module {name_mod}")
importlib.invalidate_caches()
module_pythran = importlib.import_module(name_mod)
sys.path.pop(0)
return module_pythran
spec = importlib.util.spec_from_file_location(module_name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class CachedJIT:
......@@ -254,8 +258,8 @@ class CachedJIT:
module_name = mod.module_name
print(f"Make new function to replace {func_name} ({module_name})")
path_pythran = path_cachedjit / module_name.replace(".", "__")
path_pythran.mkdir(exist_ok=True)
path_pythran = path_cachedjit / module_name.replace(".", os.path.sep)
path_pythran.mkdir(parents=True, exist_ok=True)
path_pythran = (path_pythran / func_name).with_suffix(".py")
......@@ -267,6 +271,8 @@ class CachedJIT:
else:
has_to_write = True
src = None
if has_to_write:
import_lines = [
line.split("# pythran ")[1]
......@@ -294,18 +300,26 @@ class CachedJIT:
file.write(src)
file.flush()
if src is None:
with open(path_pythran) as file:
src = file.read()
# hash from src (to produce the extension name)
hex_src = hashlib.md5(src.encode("utf8")).hexdigest()
name_mod = ".".join(
path_pythran.absolute().relative_to(path_root).with_suffix("").parts
)
module_pythran = reimport_module(name_mod, avoid_extension=False)
glob_name_ext_file = func_name + "_" + hex_src + "_*" + ext_suffix
ext_files = list(path_pythran.parent.glob(glob_name_ext_file))
if hasattr(module_pythran, "__pythran__"):
print(f"module {module_pythran.__name__} already pythranized")
self.pythran_func = getattr(module_pythran, func_name)
if has_to_build(module_pythran.__file__, path_pythran):
if not ext_files:
self.pythran_func = None
else:
path_ext = max(ext_files, key=lambda p: p.stat().st_ctime)
module_pythran = import_from_path(path_ext, name_mod)
self.pythran_func = getattr(module_pythran, func_name)
path_pythran_header = path_pythran.with_suffix(".pythran")
......@@ -315,7 +329,9 @@ class CachedJIT:
if self.process.poll() is not None:
self.compiling = False
time.sleep(0.1)
module_pythran = reimport_module(name_mod)
module_pythran = import_from_path(
self.path_extension, name_mod
)
assert hasattr(module_pythran, "__pythran__")
self.pythran_func = getattr(module_pythran, func_name)
......@@ -358,8 +374,21 @@ class CachedJIT:
file.write(header)
file.flush()
# compute the new path of the extension
hex_header = hashlib.md5(header.encode("utf8")).hexdigest()
name_ext_file = (
func_name + "_" + hex_src + "_" + hex_header + ext_suffix
)
self.path_extension = path_pythran.with_name(name_ext_file)
self.compiling = True
words_command = ["pythran", "-v", str(path_pythran)]
words_command = [
"pythran",
"-v",
str(path_pythran),
"-o",
name_ext_file,
]
if self.native:
words_command.append("-march=native")
......
import sys
import os
import numpy as np
......@@ -14,11 +15,11 @@ scheduler.nb_cpus = 2
module_name = "fluidpythran.for_test_cached_jit"
path_pythran_dir = path_cachedjit / module_name.replace(".", "__")
path_pythran_dir = path_cachedjit / module_name.replace(".", os.path.sep)
def delete_pythran_files(func_name):
for path_pythran_file in path_pythran_dir.glob(func_name + ".*"):
for path_pythran_file in path_pythran_dir.glob(func_name + "*"):
if path_pythran_file.exists():
path_pythran_file.unlink()
......
......@@ -9,7 +9,7 @@ def test_not_fluidpythranized():
path_for_test = Path(__file__).parent / "for_test_init.py"
path_output = path_for_test.parent / "_pythran"
path_output = path_for_test.parent / "__pythran__"
if path_output.exists():
shutil.rmtree(path_output)
......
......@@ -11,7 +11,7 @@ class TestsInit(unittest.TestCase):
assert path_for_test.exists()
path_output = path_for_test.parent / ("_pythran/_" + path_for_test.name)
path_output = path_for_test.parent / ("__pythran__/_" + path_for_test.name)
@classmethod
def tearDownClass(cls):
......
......@@ -7,7 +7,7 @@ from .run import run
def test_create_pythran_files():
path_dir_out = path_data_tests / "_pythran"
path_dir_out = path_data_tests / "__pythran__"
if path_dir_out.exists():
rmtree(path_dir_out)
......@@ -28,7 +28,7 @@ def test_create_pythran_files():
path_file.touch()
run()
path_file = path_data_tests / "_pythran/_classic.py"
path_file = path_data_tests / "__pythran__/_classic.py"
path_file.unlink()
run()
......
......@@ -434,7 +434,7 @@ def make_pythran_file(path_py: Path, force=False, log_level=None):
path_py = Path(path_py)
if path_py.absolute().parent.name == "_pythran":
if path_py.absolute().parent.name == "__pythran__":
logger.debug(f"skip file {path_py}")
return
if not path_py.name.endswith(".py"):
......@@ -442,7 +442,7 @@ def make_pythran_file(path_py: Path, force=False, log_level=None):
"fluidpythran only processes Python file. Cannot process {path_py}"
)
path_dir = path_py.parent / "_pythran"
path_dir = path_py.parent / "__pythran__"
path_pythran = path_dir / ("_" + path_py.name)
if not has_to_build(path_pythran, path_py) and not force:
......
......@@ -20,7 +20,7 @@ source = ./fluidpythran
data_file = .coverage/coverage
omit = */test*.py
*/_try_*.py
fluidpythran/_pythran/_for_test_init.py
fluidpythran/__pythran__/_for_test_init.py
[coverage:report]
show_missing = True
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment