Commit 02a04d25 authored by Pierre Augier's avatar Pierre Augier
Browse files

wait_for_all_extensions

parent 00e3f0c33ebd
0.1.1 (?)
---------
- :code:`wait_for_all_extensions`
- Bug fixes
0.1.0 (2018-11-23)
------------------
......
......@@ -28,6 +28,7 @@ of the modules, classes and functions.
fluidpythran.transpiler
fluidpythran.annotation
fluidpythran.cached_jit
fluidpythran.pythranizer
fluidpythran.dist
fluidpythran.run
fluidpythran.log
......
......@@ -11,6 +11,7 @@ from .annotation import Array, NDim, Type, Shape
from .aheadoftime import FluidPythran, pythran_def, make_signature
from .util import set_pythranize_at_import
from .pythranizer import wait_for_all_extensions
__all__ = [
"__version__",
......@@ -25,4 +26,5 @@ __all__ = [
"cachedjit",
"used_by_cachedjit",
"set_pythranize_at_import",
"wait_for_all_extensions",
]
......@@ -30,14 +30,14 @@ import subprocess
from .util import (
get_module_name,
compile_pythran_file,
has_to_pythranize_at_import,
import_from_path,
has_to_build,
modification_date,
name_ext_from_path_pythran,
)
from .pythranizer import compile_pythran_file, name_ext_from_path_pythran
from .annotation import (
make_signature_from_template_variables,
make_signatures_from_typehinted_func,
......@@ -229,7 +229,15 @@ class FluidPythran:
time_pythran = 0
print(f"Running fluidpythran on file {path_mod}... ", end="")
# better to do this in another process because the file is already run...
subprocess.call(["fluidpythran", "-np", str(path_mod)])
returncode = subprocess.call(
["fluidpythran", "-np", str(path_mod)]
)
if returncode != 0:
raise RuntimeError(
"fluidpythran does not manage to produce the Pythran "
f"file for {path_mod}"
)
print("Done!")
path_ext = path_pythran.with_name(
......@@ -259,7 +267,7 @@ class FluidPythran:
):
print("Launching Pythran to compile a new extension...")
self.process = compile_pythran_file(
path_pythran, name_ext_file=self.path_extension
path_pythran, name_ext_file=self.path_extension.name
)
self.is_compiling = True
self.is_compiled = False
......
......@@ -57,16 +57,15 @@ import os
import sys
import time
from .pythranizer import compile_pythran_file, ext_suffix_short
from .util import (
get_module_name,
has_to_build,
get_source_without_decorator,
path_root,
ext_suffix_short,
get_info_from_ipython,
make_hex,
compile_pythran_file,
has_to_pythranize_at_import,
import_from_path,
pythran,
......
......@@ -25,6 +25,7 @@ if implementation == "PyPy":
def rmtree(path):
shutil.rmtree(str(path))
else:
from builtins import open
......
"""Use Pythran to create extensions
===================================
User API
--------
.. autofunction:: wait_for_all_extensions
Internal API
------------
.. autofunction:: make_hex
.. autoclass:: SchedulerPopen
:members:
:private-members:
.. autofunction:: name_ext_from_path_pythran
.. autofunction:: compile_pythran_files
.. autofunction:: compile_pythran_file
"""
import multiprocessing
import subprocess
import time
from pathlib import Path
from typing import Union, Iterable, Optional
import sysconfig
import hashlib
from .compat import open, implementation
ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
# if pythran and pythran.__version__ <= "0.9.0":
# avoid a Pythran bug with -o option
# it is bad because then we do not support using many Python versions
ext_suffix_short = "." + ext_suffix.rsplit(".", 1)[-1]
def make_hex(src):
"""Produce a hash from a sting"""
return hashlib.md5(src.encode("utf8")).hexdigest()
class SchedulerPopen:
"""Limit the number of Pythran compilations performed in parallel
"""
deltat = 0.2
def __init__(self, parallel=True):
self.processes = []
if parallel:
self.nb_cpus = multiprocessing.cpu_count()
else:
self.nb_cpus = 1
def launch_popen(self, words_command, cwd=None, parallel=True):
"""Launch a program (blocking if too many processes launched)"""
if parallel:
limit = self.nb_cpus
else:
limit = 1
while len(self.processes) >= limit:
time.sleep(self.deltat)
self.processes = [
process for process in self.processes if process.poll() is None
]
if implementation == "PyPy":
cwd = str(cwd)
words_command = [str(word) for word in words_command]
process = subprocess.Popen(words_command, cwd=cwd)
self.processes.append(process)
return process
def wait_for_all_extensions(self):
"""Wait until all compilation processes are done"""
while self.processes:
time.sleep(self.deltat)
self.processes = [
process for process in self.processes if process.poll() is None
]
scheduler = SchedulerPopen()
def wait_for_all_extensions():
"""Wait until all compilation processes are done"""
scheduler.wait_for_all_extensions()
def name_ext_from_path_pythran(path_pythran):
"""Return an extension name given the path of a Pythran file"""
if path_pythran.exists():
with open(path_pythran) as file:
src = file.read()
else:
src = ""
return path_pythran.stem + "_" + make_hex(src) + ext_suffix_short
def compile_pythran_files(
paths: Iterable[Path], str_pythran_flags: str, parallel=True
):
pythran_flags = str_pythran_flags.strip().split()
for path in paths:
name_ext = name_ext_from_path_pythran(path)
words_command = ["pythran", path.name, "-o", name_ext]
words_command.extend(pythran_flags)
print("pythranize file", path)
scheduler.launch_popen(
words_command, cwd=str(path.parent), parallel=parallel
)
def compile_pythran_file(
path: Union[Path, str],
name_ext_file: Optional[str] = None,
native=True,
xsimd=True,
openmp=False,
):
if not isinstance(path, Path):
path = Path(path)
words_command = ["pythran", "-v", path.name]
if name_ext_file is not None:
words_command.extend(("-o", name_ext_file))
if native:
words_command.append("-march=native")
if xsimd:
words_command.append("-DUSE_XSIMD")
if openmp:
words_command.append("-fopenmp")
# return the process
return scheduler.launch_popen(words_command, cwd=str(path.parent))
......@@ -17,12 +17,8 @@ from glob import glob
from . import __version__
from .transpiler import make_pythran_files
from .log import logger, set_log_level
from .util import (
compile_pythran_files,
ext_suffix_short,
has_to_build,
clear_cached_extensions,
)
from .pythranizer import compile_pythran_files, ext_suffix_short
from .util import has_to_build, clear_cached_extensions
try:
import pythran
......
......@@ -8,9 +8,8 @@ try:
except ImportError:
pythran = None
from .cached_jit import path_cachedjit, modules
from .util import scheduler
from .pythranizer import scheduler, wait_for_all_extensions
scheduler.nb_cpus = 2
......@@ -85,6 +84,11 @@ def test_cachedjit_dict():
mod = modules[module_name]
cjit = mod.cachedjit_functions["func_dict"]
d = dict(a=1, b=2)
func_dict(d)
wait_for_all_extensions()
for _ in range(100):
d = dict(a=1, b=2)
func_dict(d)
......
......@@ -228,6 +228,9 @@ def make_pythran_code_functions(
logger.warning("No Pythran signature for function " + name_func)
continue
if not signatures:
raise RuntimeError("No Pythran signature for function " + name_func)
for signature in signatures:
code_pythran += f"# pythran export {signature}\n"
......
......@@ -18,22 +18,10 @@ Internal API
.. autofunction:: strip_typehints
.. autofunction:: make_hex
.. autofunction:: get_ipython_input
.. autofunction:: get_info_from_ipython
.. autoclass:: SchedulerPopen
:members:
:private-members:
.. autofunction:: name_ext_from_path_pythran
.. autofunction:: compile_pythran_files
.. autofunction:: compile_pythran_file
.. autofunction:: has_to_pythranize_at_import
.. autofunction:: import_from_path
......@@ -50,20 +38,13 @@ from datetime import datetime
import re
from pathlib import Path
import ast
import hashlib
import sysconfig
import importlib.util
from distutils.util import strtobool
import shutil
# for SchedulerPopen
import subprocess
import multiprocessing
import time
from typing import Callable, Iterable, Union, Optional
from typing import Callable
from .compat import open, implementation
import astunparse
......@@ -77,15 +58,15 @@ try:
except ImportError:
pass
path_root = Path.home() / ".fluidpythran"
ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
# if pythran and pythran.__version__ <= "0.9.0":
from .compat import implementation
from .pythranizer import (
ext_suffix,
ext_suffix_short,
name_ext_from_path_pythran,
make_hex,
)
# avoid a Pythran bug with -o option
# it is bad because then we do not support using many Python versions
ext_suffix_short = "." + ext_suffix.rsplit(".", 1)[-1]
path_root = Path.home() / ".fluidpythran"
def get_module_name(frame):
......@@ -154,11 +135,6 @@ def strip_typehints(source):
return astunparse.unparse(transformed)
def make_hex(src):
"""Produce a hash from a sting"""
return hashlib.md5(src.encode("utf8")).hexdigest()
def get_ipython_input(last=True):
"""Get the input code when called from IPython"""
ip = get_ipython()
......@@ -178,102 +154,6 @@ def get_info_from_ipython():
return src, dummy_filename
class SchedulerPopen:
"""Limit the number of Pythran compilations performed in parallel
"""
deltat = 0.2
def __init__(self, parallel=True):
self.processes = []
if parallel:
self.nb_cpus = multiprocessing.cpu_count()
else:
self.nb_cpus = 1
def launch_popen(self, words_command, cwd=None, parallel=True):
"""Launch a program (blocking if too many processes launched)"""
if parallel:
limit = self.nb_cpus
else:
limit = 1
while len(self.processes) >= limit:
time.sleep(self.deltat)
self.processes = [
process for process in self.processes if process.poll() is None
]
if implementation == "PyPy":
cwd = str(cwd)
words_command = [str(word) for word in words_command]
process = subprocess.Popen(words_command, cwd=cwd)
self.processes.append(process)
return process
scheduler = SchedulerPopen()
def name_ext_from_path_pythran(path_pythran):
if path_pythran.exists():
with open(path_pythran) as file:
src = file.read()
else:
src = ""
return path_pythran.stem + "_" + make_hex(src) + ext_suffix_short
def compile_pythran_files(
paths: Iterable[Path], str_pythran_flags: str, parallel=True
):
pythran_flags = str_pythran_flags.strip().split()
for path in paths:
name_ext = name_ext_from_path_pythran(path)
words_command = ["pythran", path.name, "-o", name_ext]
words_command.extend(pythran_flags)
print("pythranize file", path)
scheduler.launch_popen(
words_command, cwd=str(path.parent), parallel=parallel
)
def compile_pythran_file(
path: Union[Path, str],
name_ext_file: Optional[str] = None,
native=True,
xsimd=True,
openmp=False,
):
if not isinstance(path, Path):
path = Path(path)
words_command = ["pythran", "-v", str(path)]
if name_ext_file is not None:
words_command.extend(("-o", name_ext_file))
if native:
words_command.append("-march=native")
if xsimd:
words_command.append("-DUSE_XSIMD")
if openmp:
words_command.append("-fopenmp")
# return the process
return scheduler.launch_popen(words_command, cwd=str(path.parent))
_PYTHRANIZE_AT_IMPORT = None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment