Read about our upcoming Code of Conduct on this issue

Commit 551d054b authored by Pierre Augier's avatar Pierre Augier
Browse files

Tests and bugfixes

parent 0b9c895f9551
......@@ -13,6 +13,10 @@ dist*
.tox/*
.coverage/*
.pytest_cache
*.egg-info/*
**/_pythran/_pythran_*
\ No newline at end of file
**/_pythran/_pythran_*
fluidpythran/_path_data_tests.py
\ No newline at end of file
......@@ -7,3 +7,17 @@ black:
tests:
pytest -s
_tests_coverage:
mkdir -p .coverage
coverage run -p -m pytest
_report_coverage:
coverage combine
coverage report
coverage html
coverage xml
@echo "Code coverage analysis complete. View detailed report:"
@echo "file://${PWD}/.coverage/index.html"
coverage: _tests_coverage _report_coverage
import numpy as np
# pythran import numpy as np
# pythran blabla
from fluidpythran import FluidPythran
fp = FluidPythran()
class MyClass:
def __init__(self, a, b):
self.a = a
self.b = b
def compute(self, n):
a = self.a
b = self.b
if fp.is_pythranized:
result = fp.use_pythranized_block("block0")
else:
# pythran block (
# float[][] a, b;
# int n
# ) -> result
# blabla
# blibli
# pythran block (
# float[][][] a, b;
# int n
# ) -> result
result = np.zeros_like(a)
for _ in range(n):
result += a ** 2 + b ** 3
a = result
if fp.is_pythranized:
result = fp.use_pythranized_block("block1")
else:
# pythran block (
# float[][] a, b;
# int n
# ) -> (result, a)
# blabla
# blibli
# pythran block (
# float[][][] a, b;
# int n
# ) -> (result, a)
result = np.zeros_like(a)
for _ in range(n):
result += a ** 2 + b ** 3
return result
if __name__ == "__main__":
shape = 100, 100
a = np.random.rand(*shape)
b = np.random.rand(*shape)
obj = MyClass(a, b)
obj.compute(10)
if fp.is_pythranized:
ret = obj.compute(10)
fp.is_pythranized = False
ret1 = obj.compute(10)
fp.is_pythranized = True
assert np.allclose(ret, ret1)
print("allclose OK")
import numpy as np
from fluiddyn.util import mpi
from fluidpythran import FluidPythran
fp = FluidPythran()
# pythran import numpy as np
# pythran def func(
# float[][],
# float[][]
# )
# pythran def func(int[][], float[][])
@fp.pythranize
def func(a, b):
return (a * np.log(b)).max()
if __name__ == "__main__":
n0, n1 = 100, 200
a0 = np.random.rand(n0, n1)
a1 = np.random.rand(n0, n1)
result = func(a0, a1)
if mpi.nb_proc > 1:
result = mpi.comm.allreduce(result, op=mpi.MPI.MAX)
mpi.printby0(result)
a0 = (1000 * a0).astype(int)
result = func(a0, a1)
if mpi.nb_proc > 1:
result = mpi.comm.allreduce(result, op=mpi.MPI.MAX)
mpi.printby0(result)
a = 2
def myfunc():
print(a)
from fluidpythran import create_pythran_file
path = "example_classic.py"
create_pythran_file(path)
\ No newline at end of file
......@@ -26,7 +26,6 @@ class MyClass:
# float[][] a, b;
# int n
# ) -> result
# pythran block (
# float[][][] a, b;
# int n
......
......@@ -12,6 +12,7 @@ fp = FluidPythran()
# pythran def func(float[][], float[][])
# pythran def func(int[][], float[][])
@fp.pythranize
def func(a, b):
return (a * np.log(b)).max()
......
import inspect
import importlib
import os
import importlib.util
from ._version import __version__
from .toolchain import create_pythran_code, create_pythran_file
try:
from ._path_data_tests import path_data_tests
except ImportError:
pass
__all__ = [
"__version__",
"create_pythran_code",
"create_pythran_file",
"FluidPythran",
]
__all__ = ["__version__", "FluidPythran", "path_data_tests"]
class FluidPythran:
......@@ -23,18 +21,27 @@ class FluidPythran:
return
frame = inspect.stack()[1]
module_name = inspect.getmodulename(frame.filename)
module_pythran_name = "_pythran._pythran_" + module_name
module_name = inspect.getmodule(frame[0]).__name__
if module_name == "__main__":
module_name = inspect.getmodulename(frame.filename)
if "." in module_name:
package, module = module_name.rsplit(".", 1)
module_pythran = package + "._pythran._pythran_" + module
else:
module_pythran = "_pythran._pythran_" + module_name
try:
self.module_pythran = importlib.import_module(module_pythran_name)
self.module_pythran = importlib.import_module(module_pythran)
self.is_pythranized = True
except ModuleNotFoundError:
self.is_pythranized = False
else:
self.arguments_blocks = getattr(
self.module_pythran, "arguments_blocks"
)
if hasattr(self.module_pythran, "arguments_blocks"):
self.arguments_blocks = getattr(
self.module_pythran, "arguments_blocks"
)
def pythranize(self, func):
"""Decorator used for functions"""
......
__version__ = "0.0.1"
try:
from pyfiglet import figlet_format
__about__ = figlet_format("fluidpythran v" + __version__, font="big")
except:
except ImportError:
__about__ = r"""
/ _| | (_) | | | | | |
......
from tokenize import tokenize, untokenize, COMMENT, INDENT, DEDENT, STRING, NAME
import os
from datetime import datetime
from logging import DEBUG
# from token import tok_name
from token import tok_name
from io import BytesIO
import inspect
from runpy import run_path
......@@ -11,8 +13,11 @@ from pathlib import Path
import black
from .log import logger, set_log_level
def parse_py(path):
"""Parse a .py file and return data"""
blocks = []
signatures_blocks = {}
......@@ -77,7 +82,7 @@ def parse_py(path):
signature = tokval.split("# pythran block ", 1)[1]
elif in_signature:
signature += tokval[1:].strip()
if ")" in tokval and "-> (" not in tokval:
if ")" in tokval and "-> (" not in tokval or tokval.endswith(")"):
in_signature = False
if name_block not in signatures_blocks:
......@@ -98,7 +103,7 @@ def parse_py(path):
has_to_find_code_block = "in block"
code_blocks[name_block] = []
# print((tok_name[toknum], tokval))
logger.debug((tok_name[toknum], tokval))
return (
blocks,
......@@ -111,18 +116,33 @@ def parse_py(path):
def create_pythran_code(path_py):
"""Create a pythran code from a Python file"""
blocks, signatures_blocks, code_blocks, functions, signatures_func, imports = parse_py(
path_py
)
print(signatures_blocks)
if logger.isEnabledFor(DEBUG):
logger.debug(
f"""
blocks: {blocks}\n
signatures_blocks: {signatures_blocks}\n
code_blocks: {code_blocks}\n
functions: {functions}\n
signatures_func: {signatures_func}\n
imports: {imports}\n"""
)
code_pythran = "\n" + "\n".join(imports) + "\n"
# it is not optimal. We should be able to get the code of the function
# without running the module
os.environ["FLUIDPYTHRAN_COMPILING"] = "1"
mod = run_path(path_py)
del os.environ["FLUIDPYTHRAN_COMPILING"]
mod = run_path(str(path_py))
try:
del os.environ["FLUIDPYTHRAN_COMPILING"]
except KeyError:
pass
for name_func in functions:
signatures = signatures_func[name_func]
......@@ -178,32 +198,98 @@ def create_pythran_code(path_py):
if name_block in return_block:
code_pythran += f" return {return_block[name_block]}\n"
code_pythran += "# pythran export arguments_blocks\n"
code_pythran += "arguments_blocks = " + str(arguments_blocks)
if arguments_blocks:
code_pythran += "# pythran export arguments_blocks\n"
code_pythran += "arguments_blocks = " + str(arguments_blocks)
code_pythran = black.format_str(code_pythran, line_length=82)
return code_pythran
def create_pythran_file(path_py):
def modification_date(filename):
"""Get the modification date of a file"""
t = os.path.getmtime(filename)
return datetime.fromtimestamp(t)
def has_to_build(output_file, input_file):
"""Check if a file has to be (re)built"""
if not output_file.exists():
return True
mod_date_output = modification_date(output_file)
if mod_date_output < modification_date(input_file):
return True
return False
def create_pythran_file(path_py, force=False, log_level=None):
"""Create a Python file from a Python file (if necessary)"""
if log_level is not None:
set_log_level(log_level)
path_py = Path(path_py)
if path_py.name.startswith("_pythran_"):
logger.debug(f"skip file {path_py}")
return
if not path_py.name.endswith(".py"):
raise ValueError(
"fluidpythran only processes Python file. Cannot process {path_py}"
)
path_dir = path_py.parent / "_pythran"
path_pythran = path_dir / ("_pythran_" + path_py.name)
if not has_to_build(path_pythran, path_py) and not force:
logger.info(f"File {path_pythran} already up-to-date.")
return
code_pythran = create_pythran_code(path_py)
if not code_pythran:
return
print("code_pythran\n", code_pythran)
if path_pythran.exists() and not force:
with open(path_pythran) as file:
code_pythran_old = file.read()
path = Path(path_py)
if code_pythran_old == code_pythran:
logger.info(f"Code in file {path_pythran} already up-to-date.")
return
path_dir = path.parent / "_pythran"
path_dir.mkdir(exist_ok=True)
logger.debug(f"code_pythran:\n{code_pythran}")
path_pythran = path_dir / ("_pythran_" + path.name)
path_dir.mkdir(exist_ok=True)
with open(path_pythran, "w") as file:
file.write(code_pythran)
print(f"File {str(path_pythran)} written")
logger.info(f"File {str(path_pythran)} written")
return path_pythran
def create_pythran_files(paths, force=False, log_level=None):
"""Create Pythran files from a list of Python files"""
if log_level is not None:
set_log_level(log_level)
paths_out = []
for path in paths:
path_out = create_pythran_file(path, force=force)
if path_out:
paths_out.append(path_out)
if paths_out:
nb_files = len(paths_out)
if nb_files == 1:
conjug = "s"
else:
conjug = ""
logger.warning(
f"{nb_files} files created or updated need{conjug}"
" to be pythranized"
)
import numpy as np
# pythran import numpy as np
from fluidpythran import FluidPythran
fp = FluidPythran()
# pythran def func(int, float)
@fp.pythranize
def func(a, b):
return a + b
def func1(a, b):
n = 10
if fp.is_pythranized:
result = fp.use_pythranized_block("block0")
else:
# pythran block (
# float a, b;
# int n
# ) -> (result, a)
# blabla
result = np.zeros_like(a)
for _ in range(n):
result += a ** 2 + b ** 3
import logging
logger = logging.getLogger("pythran")
# Initialize logging
try:
# Set a nice colored output
from colorlog import ColoredFormatter
formatter = ColoredFormatter(
"%(log_color)s%(levelname)-8s%(reset)s %(blue)s%(message)s",
log_colors={
"DEBUG": "cyan",
"INFO": "green",
"WARNING": "yellow",
"ERROR": "red",
"CRITICAL": "red",
},
)
stream = logging.StreamHandler()
stream.setFormatter(formatter)
logger.addHandler(stream)
except ImportError:
# No color available, use default config
logging.basicConfig(format="%(levelname)s: %(message)s")
logger.warn("Disabling color, you really want to install colorlog.")
def set_log_level(level):
"""Set logging level"""
level = level.lower()
if level == "info":
level = logging.INFO
elif level == "debug":
level = logging.DEBUG
logger.setLevel(level)
set_log_level("info")
"""Command line fluidpythran
============================
"""
import argparse
from pathlib import Path
from glob import glob
from .files_maker import create_pythran_files
from .log import logger, set_log_level
doc = """
fluidpythran: Pythran code in Python files
"""
def parse_args():
parser = argparse.ArgumentParser(
description=doc, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("path", help="Path file or directory.", nargs="*")
parser.add_argument(
"-f",
"--force",
help="write the file even if it is up-to-date",
action="store_true",
)
parser.add_argument("-v", "--verbose", help="verbose mode", action="count")
return parser.parse_args()
def run():
args = parse_args()
if args.verbose is None:
set_log_level("info")
elif args.verbose > 0:
set_log_level("debug")
logger.debug(args)
path = args.path
if isinstance(path, list) and len(path) == 1:
path = path[0]
if isinstance(path, list):
paths = path
else:
path = Path(path)
if path.is_file():
paths = (path,)
elif path.is_dir():
paths = path.glob("*.py")
else:
paths = glob(str(path))
create_pythran_files(paths, force=args.force)
from .for_test_init import func, func1
def test_not_fluidpythranized():
func(1, 3.14)
func1(1.1, 2.2)
from pathlib import Path
import importlib
import unittest
from .files_maker import create_pythran_file
path_for_test = Path(__file__).parent / "for_test_init.py"
assert path_for_test.exists()
print(path_for_test)
path_output = path_for_test.parent / ("_pythran/_pythran_" + path_for_test.name)
if path_output.exists():
path_output.unlink()
create_pythran_file(path_for_test)
print("path_output", path_output)
from . import for_test_init
importlib.reload(for_test_init)
class TestsInit(unittest.TestCase):
@classmethod
def tearDownClass(cls):
path_output.unlink()
def test_fluidpythranized(self):
assert path_output.exists()