Read about our upcoming Code of Conduct on this issue

Commit bb4cc594 authored by Pierre Blanc-fatin's avatar Pierre Blanc-fatin
Browse files

Restructuration : tranpiler & pythranizer in backends

parent de5be5dd99ac
......@@ -27,8 +27,8 @@ else:
Extension("add_cython", ["add_cython.pyx"], include_dirs=include_dirs)
)
pythranBE = PythranBackend([path_sources / "add.py"])
pythranBE.make_backend_files()
pythran_be = PythranBackend()
pythran_be.make_backend_files([path_sources / "add.py"])
extensions.extend(init_pythran_extensions(".", include_dirs=include_dirs))
setup(
......
from transonic.backends.cython import CythonBackend
paths = ["/home/users/blancfat8p/work/transonic/tmp/for_test_cython.py", ]
BE = CythonBackend(paths)
BE.make_backend_files()
\ No newline at end of file
from transonic import boost
import numpy as np
@boost
def primes(nb_primes : int, a : int= 1, b = None):
n : int
len_p : int
i : int
p = np.zeros(1000)
if nb_primes > 1000:
nb_primes = 1000
len_p = 2 # The current number of elements in p.
n = 2
while len_p < nb_primes:
# Is n prime?
for i in p[:len_p]:
if n % i == 0:
break
# If no break occurred in the loop, we have a prime.
else:
p[len_p] = n
len_p += 1
n += 1
# Let's return the result in a python list:
result_as_list = [prime for prime in p[:len_p]]
return result_as_list
print(primes(10))
......@@ -6,7 +6,7 @@ from .annotation import Array, NDim, Type, Shape, Union
from .aheadoftime import Transonic, boost, include
from .util import set_compile_at_import
from .pythranizer import wait_for_all_extensions
from transonic.backends.pythranizer import wait_for_all_extensions
__all__ = [
"__version__",
......
......@@ -41,10 +41,12 @@ from .util import (
write_if_has_to_write,
)
from .pythranizer import compile_extension, name_ext_from_path_backend, ext_suffix
from .transpiler import produce_code_class
from transonic.backends.pythranizer import (
compile_extension,
name_ext_from_path_backend,
ext_suffix,
)
from transonic.backends.transpiler import produce_code_class
from .log import logger
from . import mpi
......
......@@ -272,13 +272,13 @@ def analyse_aot(code, pathfile):
code_dependance = capturex.make_code_external()
if boosted_dicts["functions"]:
func = next(iter(boosted_dicts["functions"]))
code_dependance, code_ext, jitted_dicts, spe = get_exterior_code(
code_dependance, code_ext, _, _ = get_exterior_code(
{func: code_dependance}, pathfile, classes="function", relative=True
)
code_dependance = code_dependance[func]
if boosted_dicts["classes"]:
cls = next(iter(boosted_dicts["classes"]))
code_dependance, code_ext, jitted_dicts, spe = get_exterior_code(
code_dependance, code_ext, _, _ = get_exterior_code(
{cls: code_dependance}, pathfile, classes="classe", relative=True
)
code_dependance = code_dependance[cls]
......
......@@ -408,3 +408,17 @@ def get_exterior_code(
relative,
)
return codes_dependance, code_ext, jitted_dicts, special
def get_source_with_numba(func: Callable):
"""Get the source and adapt to numba"""
src = inspect.getsource(func)
src = dedent(src)
mod = extast.parse(src)
for node in mod.body:
node.decorator_list[0].keywords = [
ast.keyword("nopython", value=ast.Str("True")),
ast.keyword("nogil", value=ast.Str("True")),
]
src = "\nfrom numba import jit \n" + extast.unparse(mod) + "\n"
return src
......@@ -7,3 +7,13 @@
pythran
"""
from transonic.analyses import analyse_aot
from .pythran import PythranBackend
from .numba import NumbaBackend
from .cython import CythonBackend
backends = dict(
pythran=PythranBackend(), numba=NumbaBackend(), cython=CythonBackend()
)
......@@ -6,8 +6,8 @@ from transonic.analyses import extast
class Backend:
def __init__(self, paths_py=None):
self.paths_py = paths_py
def __init__(self):
pass
def get_code_function(self, fdef):
......
......@@ -22,11 +22,11 @@ from .backend import Backend
class CythonBackend(Backend):
def __init__(self, paths_py):
super(CythonBackend, self).__init__(paths_py)
def __init__(self):
pass
def make_backend_files(self):
for path in self.paths_py:
def make_backend_files(self, paths_py):
for path in paths_py:
self.make_cython_file(path)
def make_cython_file(self, path_py):
......@@ -92,17 +92,20 @@ class CythonBackend(Backend):
# )
anns = annotations["comments"][func_name]
print(anns)
print_dumped(fdef.decorator_list)
# change annotations
for name in fdef.args.args:
if name.annotation:
name.id = name.annotation.id + " " + name.id
# change type hints into cdef
for index, node in enumerate(fdef.body):
if isinstance(node, ast.AnnAssign):
cdef = "cdef " + node.annotation.id + " " + node.target.id
fdef.body[index] = extast.CommentLine(s=cdef)
code.append(extast.unparse(fdef))
if fdef.decorator_list:
for name in fdef.args.args:
if name.annotation:
name.id = name.annotation.id + " " + name.id
# change type hints into cdef
for index, node in enumerate(fdef.body):
if isinstance(node, ast.AnnAssign):
cdef = "cdef " + node.annotation.id + " " + node.target.id
fdef.body[index] = extast.CommentLine(s=cdef)
code.append(extast.unparse(fdef))
return code, code_ext
......@@ -7,6 +7,7 @@ from pathlib import Path
from transonic.analyses import analyse_aot
from transonic.analyses import extast
from transonic.analyses.util import get_source_with_numba
from transonic.util import (
has_to_build,
......@@ -21,13 +22,13 @@ from .backend import Backend
class NumbaBackend(Backend):
def __init__(self, paths_py):
super.__init__(paths_py)
def __init__(self):
pass
def make_backend_files(self):
def make_backend_files(self, paths_py):
"""Create numba files from a list of Python files"""
paths_out = []
for path in self.paths_py:
for path in paths_py:
path_out = self.make_numba_file(path)
if path_out:
paths_out.append(path_out)
......@@ -109,4 +110,22 @@ class NumbaBackend(Backend):
code_module, path_py
)
print(boosted_dicts)
code = ["\n" + code_dependance + "\n"]
for func_name, fdef in boosted_dicts["functions"].items():
signatures_func = set()
try:
ann = annotations["functions"][func_name]
except KeyError:
pass
# else:
# typess = compute_pythran_types_from_valued_types(ann.values())
# for types in typess:
# signatures_func.add(
# f"# pythran export {func_name}({', '.join(types)})"
# )
# code.append(get_source_with_numba(fdef))
return None, None
......@@ -38,11 +38,12 @@ from transonic.util import (
class PythranBackend(Backend):
def __init__(self, paths_py=None):
super().__init__(paths_py)
def __init__(self):
pass
def make_backend_files(
self,
paths_py,
force=False,
log_level=None,
mocked_modules: Optional[Iterable] = None,
......@@ -63,7 +64,7 @@ class PythranBackend(Backend):
logger.set_level(log_level)
paths_out = []
for path in self.paths_py:
for path in paths_py:
path_out = self.make_pythran_file(path, force=force)
if path_out:
paths_out.append(path_out)
......
......@@ -33,9 +33,9 @@ import sys
import os
from datetime import datetime
from . import mpi
from .mpi import Path, PathSeq
from .log import logger
from .. import mpi
from ..mpi import Path, PathSeq
from ..log import logger
ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
......
......@@ -25,16 +25,16 @@ from pathlib import Path
from typing import Iterable, Optional
from .log import logger
from .annotation import compute_pythran_types_from_valued_types
from .util import (
from ..log import logger
from ..annotation import compute_pythran_types_from_valued_types
from ..util import (
has_to_build,
get_source_without_decorator,
format_str,
write_if_has_to_write,
)
from transonic.backends.pythran import PythranBackend
from . import backends
def produce_code_class(cls, jit=False):
......@@ -212,6 +212,106 @@ def produce_code_class_func(cls, func_name, jit=False):
return python_code
def make_pythran_file(path_py: Path, force=False, log_level=None):
"""Create a Python file from a Python file (if necessary)"""
if log_level is not None:
logger.set_level(log_level)
path_py = Path(path_py)
if not path_py.exists():
raise FileNotFoundError(f"Input file {path_py} not found")
if path_py.absolute().parent.name == "__pythran__":
logger.debug(f"skip file {path_py}")
return
if not path_py.name.endswith(".py"):
raise ValueError(
"transonic only processes Python file. Cannot process {path_py}"
)
path_dir = path_py.parent / "__pythran__"
path_pythran = path_dir / path_py.name
if not has_to_build(path_pythran, path_py) and not force:
logger.warning(f"File {path_pythran} already up-to-date.")
return
code_pythran, code_ext = backends["pythran"].make_pythran_code(path_py)
if not code_pythran:
return
for file_name, code in code_ext["function"].items():
path_ext_file = path_dir / (file_name + ".py")
write_if_has_to_write(path_ext_file, code, logger.info)
for file_name, code in code_ext["classe"].items():
path_ext_file = path_dir.parent / "__pythran__" / (file_name + ".py")
write_if_has_to_write(path_ext_file, code, logger.info)
if path_pythran.exists() and not force:
with open(path_pythran) as file:
code_pythran_old = file.read()
if code_pythran_old == code_pythran:
logger.warning(f"Code in file {path_pythran} already up-to-date.")
return
logger.debug(f"code_pythran:\n{code_pythran}")
path_dir.mkdir(exist_ok=True)
with open(path_pythran, "w") as file:
file.write(code_pythran)
logger.info(f"File {path_pythran} written")
return path_pythran
def make_backend_files(
paths: Iterable[Path],
force=False,
log_level=None,
mocked_modules: Optional[Iterable] = None,
backend=None,
):
"""Create Pythran files from a list of Python files"""
assert backend is None
if mocked_modules is not None:
warn(
"The argument mocked_modules is deprecated. "
"It is now useless for Transonic.",
DeprecationWarning,
)
if log_level is not None:
logger.set_level(log_level)
paths_out = []
for path in paths:
path_out = make_pythran_file(path, force=force)
if path_out:
paths_out.append(path_out)
if paths_out:
nb_files = len(paths_out)
if nb_files == 1:
conjug = "s"
else:
conjug = ""
logger.warning(
f"{nb_files} files created or updated need{conjug}"
" to be pythranized"
)
return paths_out
class _MyMock(Mock):
@classmethod
def __getattr__(cls, name):
......
......@@ -62,7 +62,7 @@ try:
except ImportError:
np = None
from .pythranizer import compile_extension, ext_suffix
from transonic.backends.pythranizer import compile_extension, ext_suffix
from .util import (
get_module_name,
......
......@@ -18,7 +18,11 @@ import sys
from . import __version__
from .backends.pythran import PythranBackend
from .log import logger
from .pythranizer import compile_extensions, ext_suffix, wait_for_all_extensions
from transonic.backends.pythranizer import (
compile_extensions,
ext_suffix,
wait_for_all_extensions,
)
from .util import has_to_build, clear_cached_extensions
try:
......@@ -80,8 +84,10 @@ def run():
logger.error(f"No input file found (args.path = {args.path})")
sys.exit(1)
pythranBE = PythranBackend(paths)
pythranBE.make_backend_files(force=args.force)
from .backends import backends
for key, backend in backends.items():
backend.make_backend_files(paths_py=paths)
if not pythran or args.no_pythran:
return
......
......@@ -17,8 +17,8 @@ setup_dir = path_data_tests / "test_packaging"
def setup_module():
os.chdir(setup_dir)
transonic_src_paths = [setup_dir / "add.py"]
pythranBE = PythranBackend(transonic_src_paths)
pythranBE.make_backend_files()
pythran_be = PythranBackend()
pythran_be.make_backend_files(transonic_src_paths)
@pytest.mark.skipif(not path_data_tests.exists(), reason="no data tests")
......
......@@ -42,8 +42,8 @@ def test_detect_pythran_extensions():
"boosted_func_use_import.py",
]
pythranBE = PythranBackend((path_data_tests / name for name in names))
pythranBE.make_backend_files()
pythranBE = PythranBackend()
pythranBE.make_backend_files((path_data_tests / name for name in names))
ext_names = detect_pythran_extensions(path_data_tests)
# -2 files (no_pythran.py and assign_fun_jit.py) +2 files (__func__exterior_import_boost.py and __func__exterior_import_boost_2.py)
assert len(ext_names) == len(names)
......
......@@ -3,13 +3,12 @@ from shutil import rmtree
from . import Transonic, mpi
from .mpi import Path
from .pythranizer import wait_for_all_extensions
from .backends.pythranizer import wait_for_all_extensions
def test_not_transonified():
path_for_test = Path(__file__).parent / "for_test_init.py"
path_output = path_for_test.parent / "__pythran__"
if path_output.exists() and mpi.rank == 0:
......
......@@ -9,7 +9,7 @@ except ImportError:
pythran = False
from .backends.pythran import PythranBackend
from .backends.transpiler import make_pythran_file
from .util import has_to_compile_at_import, ext_suffix, name_ext_from_path_backend
from .aheadoftime import modules
from . import mpi
......@@ -74,9 +74,7 @@ class TestsInit(unittest.TestCase):
print(mpi.rank, "before make_pythran_file(self.path_for_test)", flush=1)
if mpi.rank == 0:
pythranBE = PythranBackend()
print(self.path_for_test)
pythranBE.make_pythran_file(self.path_for_test)
make_pythran_file(self.path_for_test)
print(mpi.rank, "after make_pythran_file(self.path_for_test)", flush=1)
mpi.barrier()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment