Commit 357eb6af authored by Pierre Augier's avatar Pierre Augier
Browse files

fluidpythran_cl to split _pythran-fluid and mpi

--HG--
branch : dev
parent 39de205b6590
......@@ -3,7 +3,7 @@ develop:
pip install -e .[test]
black:
black -l 82 fluidpythran
black -l 82 fluidpythran fluidpythran_cl
tests:
pytest fluidpythran data_tests/ipynb
......
......@@ -301,9 +301,7 @@ def compute_pythran_types_from_types(types, **kwargs):
if "][" in _type:
# C style: we try to rewrite it in Cython style
base, dims = _type.split("[", 1)
dims = ", ".join(
[_ or ":" for _ in dims[:-1].split("][")]
)
dims = ", ".join([_ or ":" for _ in dims[:-1].split("][")])
_type = base + "[" + dims + "]"
elif _type.endswith("[]"):
_type = _type[:-2] + "[:]"
......
......@@ -334,13 +334,16 @@ class CachedJIT:
if not exports:
return
mpi.barrier()
if path_pythran_header.exists():
# get the old signature(s)
exports_old = None
if mpi.rank == 0:
with open(path_pythran_header) as file:
exports_old = [export.strip() for export in file.readlines()]
exports_old = [
export.strip() for export in file.readlines()
]
exports_old = mpi.bcast(exports_old)
# FIXME: what do we do with the old signatures?
......@@ -358,6 +361,8 @@ class CachedJIT:
# compute the new path of the extension
hex_header = make_hex(header)
# to be sure (possible bug...)
hex_header = mpi.bcast(hex_header)
name_ext_file = (
func_name + "_" + hex_src + "_" + hex_header + ext_suffix
)
......
......@@ -5,7 +5,7 @@
import os
if "__FLUIDPYTHRAN_NO_MPI" in os.environ:
if "FLUIDPYTHRAN_NO_MPI" in os.environ:
nb_proc = 1
rank = 0
else:
......@@ -27,8 +27,6 @@ else:
if nb_proc > 1:
comm = _mpi.comm
from .log import logger
def barrier():
if nb_proc > 1:
......
......@@ -70,7 +70,9 @@ class SchedulerPopen:
while len(self.processes) >= limit:
time.sleep(self.deltat)
self.processes = [
process for process in self.processes if process.is_alive_root()
process
for process in self.processes
if process.is_alive_root()
]
mpi.barrier()
......@@ -135,7 +137,6 @@ class SchedulerPopen:
process = None
if mpi.rank == 0:
os.environ["__FLUIDPYTHRAN_NO_MPI"] = "1"
process = subprocess.Popen(words_command, cwd=cwd)
process = mpi.ShellProcessMPI(process)
......
......@@ -175,4 +175,6 @@ def pythran_fluid():
path_tmp.rename(path_out)
logger.info(f"file {Path.cwd() / path_out.name} created")
else:
logger.error(f"file {Path.cwd() / path_tmp.name} has not been created by Pythran")
logger.error(
f"file {Path.cwd() / path_tmp.name} has not been created by Pythran"
)
......@@ -13,6 +13,7 @@ dist.can_import_pythran = True
shutil.rmtree(path_data_tests / "__pythran__", ignore_errors=True)
@pytest.mark.skipif(nb_proc > 1, reason="No dist in MPI")
def test_detect_pythran_extensions():
......@@ -34,6 +35,7 @@ def test_detect_pythran_extensions():
ext_names = detect_pythran_extensions(path_data_tests)
assert len(ext_names) == len(names) - 1
@pytest.mark.skipif(nb_proc > 1, reason="No dist in MPI")
def test_modification_date():
......
......@@ -56,11 +56,7 @@ try:
except ImportError:
pass
from .pythranizer import (
ext_suffix,
name_ext_from_path_pythran,
make_hex,
)
from .pythranizer import ext_suffix, name_ext_from_path_pythran, make_hex
path_root = Path.home() / ".fluidpythran"
......@@ -186,10 +182,7 @@ def import_from_path(path: Path, module_name: str):
if module_name in sys.modules:
module = sys.modules[module_name]
if (
module.__file__.endswith(ext_suffix)
and Path(module.__file__) == path
):
if module.__file__.endswith(ext_suffix) and Path(module.__file__) == path:
return module
spec = importlib.util.spec_from_file_location(module_name, str(path))
......
"""Command line _pythran-fluid
==============================
Internal API
------------
.. autofunction:: main
"""
import subprocess
import sys
import logging
from pathlib import Path
import sysconfig
logger = logging.getLogger("pythran_fluid")
logger.setLevel(logging.INFO)
ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
def main():
"""Minimal layer above the Pythran commandline"""
assert sys.argv[0].endswith("_pythran-fluid")
print("NEW")
args = sys.argv[1:]
path = Path.cwd() / args[0]
logger.info(f"Pythranize {path}")
if "-o" in args:
index_output = args.index("-o") + 1
name_out = args[index_output]
name_out_base = name_out.split(".", 1)[0]
name_tmp = name_out_base + ".tmp"
args[index_output] = name_tmp
args.insert(0, "pythran")
subprocess.call(args)
path_tmp = Path(name_tmp)
if path_tmp.exists():
path_out = path_tmp.with_suffix(ext_suffix)
path_tmp.rename(path_out)
logger.info(f"file {Path.cwd() / path_out.name} created")
else:
logger.error(
f"file {Path.cwd() / path_tmp.name} has not been created by Pythran"
)
......@@ -32,6 +32,6 @@ setup(
packages=find_packages(exclude=["doc"]),
entry_points={
"console_scripts": ["fluidpythran = fluidpythran.run:run",
"_pythran-fluid = fluidpythran.run:pythran_fluid"]
"_pythran-fluid = fluidpythran_cl.pythran_fluid:main"]
},
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment