Read about our upcoming Code of Conduct on this issue

Commit 4ee2848d authored by Pierre Augier's avatar Pierre Augier
Browse files

Reopen dev branch for pythran_def for methods

--HG--
branch : dev
......@@ -10,6 +10,8 @@ doc/**.zip
**/.ipynb_checkpoints/*
*.so
build/*
dist/*.tar.gz
dist*
......@@ -25,6 +27,6 @@ doc/ipynb/*.rst
*.egg-info/*
**/__pythran__/_*
**/__pythran__/*
fluidpythran/_path_data_tests.py
\ No newline at end of file
......@@ -3,3 +3,5 @@ e6747f29d9a9e1a00d71837c1cc2c05730d1a60e 0.0.1
1b81afabb2c802af0fa33c98e8341b33ce19100f 0.0.8
e6bb435e2a4d9b7c8465201507ed9fce3933f902 0.0.9
f365e250af615922db7a362b27e1af40695a3491 0.1.0
18d7cc6795ae36a69afbf66b0cb71caa11b3f837 0.1.2
7cc6437cc692187c9dcb44fa673f670283234493 0.1.3
......@@ -5,8 +5,14 @@ Future
- :code:`pythran_def` for simple methods (without assignation to attributes
and call of other methods)
0.1.2 (?)
---------
0.1.3 (2018-12-04)
------------------
- Lock file during Pythran compilation
- :code:`__name__` and :code:`__doc__` preserved by decorators
0.1.2 (2018-12-03)
------------------
- Private command line :code:`_pythran-fluid` to call Pythran
- MPI aware (only process rank == 0 doing IO and compilation)
......
......@@ -15,15 +15,20 @@ FluidPythran: easily speedup your Python code with Pythran
:target: https://codecov.io/bb/fluiddyn/fluidpythran/branch/default/
:alt: Code coverage
**Documentation**: https://fluidpythran.readthedocs.io
.. warning ::
FluidPythran is in an early stage. Remarks and suggestions are very
welcome.
FluidPythran starts to be used in `FluidSim
<https://bitbucket.org/fluiddyn/fluidsim>`_ (for example in `this file
<https://bitbucket.org/fluiddyn/fluidsim/src/default/fluidsim/base/time_stepping/pseudo_spect.py>`_).
FluidPythran is used in `FluidSim
<https://bitbucket.org/fluiddyn/fluidsim>`_ (see examples for `blocks
<https://bitbucket.org/fluiddyn/fluidsim/src/default/fluidsim/base/time_stepping/pseudo_spect.py>`_,
`@pythran_def
<https://bitbucket.org/fluiddyn/fluidsim/src/default/fluidsim/operators/operators3d.py>`_
and `@cachedjit
<https://bitbucket.org/fluiddyn/fluidsim/src/default/fluidsim/solvers/plate2d/output/correlations_freq.py>`_).
FluidPythran is a pure Python package (requiring Python >= 3.6 or Pypy3) to
help to write Python code that *can* use `Pythran
......@@ -105,6 +110,9 @@ Installation
pip install fluidpythran
The environment variable :code:`FLUIDPYTHRAN_DIR` can be set to control where
the cached files are saved.
A short tour of FluidPythran syntaxes
-------------------------------------
......@@ -315,7 +323,7 @@ setup.py like this:
here = Path(__file__).parent.absolute()
paths = ["fluidsim/base/time_stepping/pseudo_spect.py"]
make_pythran_files([here / path for path in paths])
make_pythran_files([here / path for path in paths], mocked_modules=["h5py"])
Note that the function :code:`make_pythran_files` does not use Pythran.
Compiling the associated Pythran file can be done if wanted (see for example
......
......@@ -19,4 +19,21 @@ If you don't like generic templating, you can also just write
.. literalinclude:: type_hints_notemplate.py
Yes, this one is neat!
\ No newline at end of file
Yes, this one is neat!
Note that one can also just write Pythran type-string in type annotations::
@pythran_def
def myfunc(a: "float[3, :]", b: float):
...
Finally, array types with only one number of dimension can simply be
defined like this::
from fluidpythran import Array
A = Array[float, "3d"]
Which has actually the same effect as::
A = "float[:, :, :]"
......@@ -59,11 +59,11 @@
Let's first clean all FluidPythran cache files...
%% Cell type:code id: tags:
``` python
paths = Path("__pythran__").glob("_tmp*.*")
paths = Path("__pythran__").glob("tmp*.*")
for path in paths:
path.unlink()
```
%% Cell type:markdown id: tags:
......
__version__ = "0.1.2"
__version__ = "0.1.3"
try:
from pyfiglet import figlet_format
......
......@@ -27,6 +27,8 @@ import inspect
import time
import subprocess
import os
import functools
import sys
from .util import (
get_module_name,
......@@ -36,7 +38,11 @@ from .util import (
modification_date,
)
from .pythranizer import compile_pythran_file, name_ext_from_path_pythran
from .pythranizer import (
compile_pythran_file,
name_ext_from_path_pythran,
ext_suffix,
)
from .annotation import (
make_signature_from_template_variables,
......@@ -79,7 +85,7 @@ def _get_fluidpythran_calling_module(index_frame: int = 2):
if module_name in modules:
fp = modules[module_name]
if (
fp._created_while_transpiling != is_transpiling
fp.is_transpiling != is_transpiling
or fp._pythranize_at_import_at_creation
!= has_to_pythranize_at_import()
or hasattr(fp, "path_mod")
......@@ -196,7 +202,7 @@ class FluidPythran:
self.names_template_variables = {}
self._created_while_transpiling = is_transpiling
self.is_transpiling = is_transpiling
if is_transpiling:
self.functions = {}
......@@ -218,11 +224,11 @@ class FluidPythran:
module_short_name = module_name
module_pythran_name = ""
module_pythran_name += "__pythran__._" + module_short_name
module_pythran_name += "__pythran__." + module_short_name
self.path_mod = path_mod = Path(frame.filename)
self.path_pythran = path_pythran = (
path_mod.parent / "__pythran__" / ("_" + module_short_name + ".py")
path_mod.parent / "__pythran__" / (module_short_name + ".py")
)
path_ext = None
......@@ -240,7 +246,13 @@ class FluidPythran:
# better to do this in another process because the file is already run...
os.environ["FLUIDPYTHRAN_NO_MPI"] = "1"
returncode = subprocess.call(
["fluidpythran", "-np", str(path_mod)]
[
sys.executable,
"-m",
"fluidpythran.run",
"-np",
str(path_mod),
]
)
del os.environ["FLUIDPYTHRAN_NO_MPI"]
returncode = mpi.bcast(returncode)
......@@ -288,6 +300,12 @@ class FluidPythran:
self.is_compiled = False
self.is_transpiled = True
if not path_ext.exists() and not self.is_compiling:
path_ext_alt = path_pythran.with_suffix(ext_suffix)
if path_ext_alt.exists():
self.path_extension = path_ext = path_ext_alt
if path_ext.exists() and not self.is_compiling:
self.module_pythran = import_from_path(
self.path_extension, module_pythran_name
......@@ -338,7 +356,7 @@ class FluidPythran:
func_tmp = func
if self.is_compiling:
return CheckCompiling(self, func_tmp)
return functools.wraps(func)(CheckCompiling(self, func_tmp))
return func_tmp
......
......@@ -177,24 +177,35 @@ class ArrayMeta(type):
def __getitem__(self, parameters):
if not isinstance(parameters, tuple):
parameters = (parameters,)
dtype = None
ndim = None
params_filtered = []
for param in parameters:
if isinstance(param, (Type, type)):
if dtype is not None:
raise ValueError
raise ValueError(
"Array should be defined with only one variable defining "
"the types. For more than one type, use "
"for example Type(float, int)"
)
dtype = param
if isinstance(param, NDim):
if ndim is not None:
raise ValueError
raise ValueError(
"Array should be defined with only "
"one NDim. For more than one dimension, use "
"for example NDim(2, 3)."
)
ndim = param
if (
isinstance(param, str)
and param[-1] == "d"
and param[0].isnumeric()
and param[:-1].isnumeric()
):
try:
tmp = int(param[:-1])
......@@ -202,7 +213,11 @@ class ArrayMeta(type):
pass
else:
if ndim is not None:
raise ValueError
raise ValueError(
"Array should be defined with only "
"one string fixing the number of dimension. "
"Use for example NDim(2, 3)."
)
param = ndim = NDim(
tmp, _fp=_get_fluidpythran_calling_module()
)
......@@ -212,6 +227,12 @@ class ArrayMeta(type):
params_filtered.append(param)
if dtype is None:
raise ValueError("No way to determine the dtype of the array")
if ndim is None:
raise ValueError("No way to determine the ndim of the array")
parameters = {p.__name__: p for p in params_filtered}
ArrayBis = type(
......
import numpy as np
# pythran import numpy as np
......
......@@ -59,6 +59,7 @@ import os
import sys
import time
from distutils.util import strtobool
from functools import wraps
try:
import numpy as np
......@@ -88,7 +89,9 @@ modules = {}
path_cachedjit = mpi.Path(path_root) / "__cachedjit__"
path_cachedjit.mkdir(parents=True, exist_ok=True)
if mpi.rank == 0:
path_cachedjit.mkdir(parents=True, exist_ok=True)
mpi.barrier()
_COMPILE_CACHEDJIT = strtobool(os.environ.get("FLUID_COMPILE_CACHEDJIT", "True"))
......@@ -252,7 +255,10 @@ class CachedJIT:
module_name = mod.module_name
path_pythran = path_cachedjit / module_name.replace(".", os.path.sep)
path_pythran.mkdir(parents=True, exist_ok=True)
if mpi.rank == 0:
path_pythran.mkdir(parents=True, exist_ok=True)
mpi.barrier()
path_pythran = (path_pythran / func_name).with_suffix(".py")
path_pythran_header = path_pythran.with_suffix(".pythran")
......@@ -389,6 +395,7 @@ class CachedJIT:
module_pythran = import_from_path(path_ext, name_mod)
self.pythran_func = getattr(module_pythran, func_name)
@wraps(func)
def type_collector(*args, **kwargs):
if self.compiling:
......
......@@ -59,9 +59,11 @@ class ShellProcessMPI:
def is_alive_root(self):
return self.process.poll() is None
PathSeq = Path
if nb_proc > 1:
class PathMPI(type(Path())):
def exists(self):
respons = None
......@@ -74,16 +76,11 @@ if nb_proc > 1:
super().unlink()
comm.barrier()
def mkdir(self, *args, **kwargs):
if rank == 0:
super().mkdir(*args, **kwargs)
comm.barrier()
Path = PathMPI
def has_to_build(output_file: Path, input_file: Path):
from . import util
ret = None
if rank == 0:
ret = util.has_to_build(output_file, input_file)
......@@ -91,6 +88,7 @@ if nb_proc > 1:
def modification_date(filename):
from . import util
ret = None
if rank == 0:
ret = util.modification_date(filename)
......
......@@ -29,6 +29,7 @@ import time
from typing import Union, Iterable, Optional
import sysconfig
import hashlib
import sys
from .compat import open, implementation
from . import mpi
......@@ -135,7 +136,12 @@ class SchedulerPopen:
if openmp:
update_flags("-fopenmp")
words_command = ["_pythran-fluid", path.name]
words_command = [
sys.executable,
"-m",
"fluidpythran_cl.run_pythran",
path.name,
]
if name_ext_file is None:
name_ext_file = name_ext_from_path_pythran(path)
......@@ -175,7 +181,7 @@ def compile_pythran_files(
paths: Iterable[Path], str_pythran_flags: str, parallel=True
):
for path in paths:
print("pythranize file", path)
print("Schedule pythranization of file", path)
scheduler.compile_with_pythran(
path, str_pythran_flags=str_pythran_flags, parallel=parallel
)
......
......@@ -8,15 +8,11 @@ Internal API
.. autofunction:: parse_args
.. autofunction:: pythran_fluid
"""
import argparse
from pathlib import Path
from glob import glob
import subprocess
import sys
from . import __version__
from .transpiler import make_pythran_files
......@@ -87,7 +83,7 @@ def run():
pythran_paths = []
for path in paths:
path = Path(path)
pythran_path = path.parent / "__pythran__" / ("_" + path.name)
pythran_path = path.parent / "__pythran__" / path.name
ext_path = pythran_path.with_suffix(ext_suffix)
if has_to_build(ext_path, pythran_path):
pythran_paths.append(pythran_path)
......@@ -144,3 +140,7 @@ def parse_args():
)
return parser.parse_args()
if __name__ == "__main__":
run()
import shutil
import pytest
......
import importlib
from . import FluidPythran, mpi
......
import importlib
import unittest
import os
......@@ -28,7 +27,7 @@ class TestsInit(unittest.TestCase):
assert path_for_test.exists()
path_pythran = path_for_test.parent / ("__pythran__/_" + path_for_test.name)
path_pythran = path_for_test.parent / "__pythran__" / path_for_test.name
path_ext = path_pythran.with_name(name_ext_from_path_pythran(path_pythran))
@classmethod
......
......@@ -60,7 +60,7 @@ def test_create_pythran_classic():
run()
path_file_pythran = path_data_tests / "__pythran__/_classic.py"
path_file_pythran = path_data_tests / "__pythran__/classic.py"
path_file_pythran.unlink()
print("after unlink")
......
......@@ -466,7 +466,7 @@ def make_pythran_file(
)
path_dir = path_py.parent / "__pythran__"
path_pythran = path_dir / ("_" + path_py.name)
path_pythran = path_dir / path_py.name
if not has_to_build(path_pythran, path_py) and not force:
logger.info(f"File {path_pythran} already up-to-date.")
......
......@@ -58,7 +58,10 @@ except ImportError:
from .pythranizer import ext_suffix, name_ext_from_path_pythran, make_hex
path_root = Path.home() / ".fluidpythran"
path_root = Path(
os.environ.get("FLUIDPYTHRAN_DIR", Path.home() / ".fluidpythran")
)
def get_module_name(frame):
......@@ -97,6 +100,18 @@ def has_to_build(output_file: Path, input_file: Path):
def get_source_without_decorator(func: Callable):
"""Get the source of a function without its decorator"""
src = inspect.getsource(func)
# de-indent the code
nb_spaces = 0
for c in src:
if not c.isspace():
break
nb_spaces += 1
if nb_spaces > 0:
lines = [line[nb_spaces:] for line in src.split("\n")]
src = "\n".join(lines)
return strip_typehints(re.sub(r"@.*?\sdef\s", "def ", src))
......@@ -235,7 +250,7 @@ def clear_cached_extensions(module_name: str, force: bool = False):
relative_path = Path(relative_path)
path_pythran = relative_path.parent / (
"__pythran__/_" + relative_path.name + ".py"
"__pythran__/" + relative_path.name + ".py"
)
path_ext = path_pythran.with_name(name_ext_from_path_pythran(path_pythran))
......@@ -254,7 +269,7 @@ def clear_cached_extensions(module_name: str, force: bool = False):
force=force,
):
print(f"Remove directory {path_pythran_dir_jit}")
shutil.rmtree(path_pythran_dir_jit)
shutil.rmtree(str(path_pythran_dir_jit))
if path_pythran.exists() or path_ext.exists():
if query_yes_no(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment