# HG changeset patch # User paugier <pierre.augier@univ-grenoble-alpes.fr> # Date 1708433298 -3600 # Tue Feb 20 13:48:18 2024 +0100 # Node ID b3decb86cc35af45c43859393575bbf73f8ffa6f # Parent d07c8128e3bbcffb5e447dd5df87f7f8e90a0f5f [project.entry-points."fluidimage.executors"] diff --git a/pyproject.toml b/pyproject.toml --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,15 @@ opencv = ["opencv-python"] # qt = ["PySide6"] +[project.entry-points."fluidimage.executors"] +exec_sequential = "fluidimage.executors.exec_sequential" +exec_async = "fluidimage.executors.exec_async" +exec_async_sequential = "fluidimage.executors.exec_async_sequential" +multi_exec_async = "fluidimage.executors.multi_exec_async" +exec_async_multi = "fluidimage.executors.exec_async_multiproc" +exec_async_servers = "fluidimage.executors.exec_async_servers" +exec_async_servers_threading = "fluidimage.executors.exec_async_servers_threading" + [tool.meson-python.args] setup = ['-Doptimization=3'] diff --git a/src/fluidimage/executors/__init__.py b/src/fluidimage/executors/__init__.py --- a/src/fluidimage/executors/__init__.py +++ b/src/fluidimage/executors/__init__.py @@ -25,7 +25,15 @@ """ +import importlib import os +import sys + +if sys.version_info < (3, 10): + from importlib_metadata import entry_points, EntryPoint +else: + from importlib.metadata import entry_points, EntryPoint + import trio @@ -38,24 +46,74 @@ os.register_at_fork(after_in_child=afterfork) from .base import ExecutorBase -from .exec_async import ExecutorAsync -from .exec_async_multiproc import ExecutorAsyncMultiproc -from .exec_async_sequential import ExecutorAsyncSequential -from .exec_async_servers import ( - ExecutorAsyncServers, - ExecutorAsyncServersThreading, -) -from .exec_sequential import ExecutorSequential -from .multi_exec_async import MultiExecutorAsync + + +_entry_points = None + + +def get_entry_points(reload=False, ndim=None, sequential=None): + """Discover the executors installed""" + global _entry_points + if _entry_points is None or reload: + _entry_points = entry_points(group="fluidimage.executors") + + if not _entry_points: + raise RuntimeError("No executor were found.") + + return _entry_points + + +def get_executor_names(): + """Get available executor names""" + return set(entry_point.name for entry_point in get_entry_points()) + + +def _get_module_fullname_from_name(name): + """Get the module name from an executor name + + Parameters + ---------- + + name : str + Name of an executor. -executors = { - "exec_sequential": ExecutorSequential, - "exec_async": ExecutorAsync, - "exec_async_sequential": ExecutorAsyncSequential, - "multi_exec_async": MultiExecutorAsync, - "exec_async_multi": ExecutorAsyncMultiproc, - "exec_async_servers": ExecutorAsyncServers, - "exec_async_servers_threading": ExecutorAsyncServersThreading, -} + """ + entry_points = get_entry_points() + selected_entry_points = entry_points.select(name=name) + if len(selected_entry_points) == 0: + raise ValueError(f"Cannot find an executor for {name = }. {entry_points}") + elif len(selected_entry_points) > 1: + logging.warning( + f"{len(selected_entry_points)} plugins were found for {name = }" + ) + + return selected_entry_points[name].value + + +def import_executor_class(name): + """Import an executor class. + + Parameters + ---------- -__all__ = ["ExecutorBase", "executors"] + name : str + Executor name. + + Returns + ------- + + The corresponding executor class. + + """ + + if isinstance(name, EntryPoint): + module_fullname = name.value + name = name.name + else: + module_fullname = _get_module_fullname_from_name(name) + + mod = importlib.import_module(module_fullname) + return mod.Executor + + +__all__ = ["ExecutorBase", "get_executor_names", "import_executor_class"] diff --git a/src/fluidimage/executors/exec_async.py b/src/fluidimage/executors/exec_async.py --- a/src/fluidimage/executors/exec_async.py +++ b/src/fluidimage/executors/exec_async.py @@ -327,3 +327,6 @@ ) await trio.sleep(self.sleep_time) + + +Executor = ExecutorAsync diff --git a/src/fluidimage/executors/exec_async_multiproc.py b/src/fluidimage/executors/exec_async_multiproc.py --- a/src/fluidimage/executors/exec_async_multiproc.py +++ b/src/fluidimage/executors/exec_async_multiproc.py @@ -132,3 +132,6 @@ if work.output_queue is not None: work.output_queue[key] = ret self.nb_working_workers_cpu -= 1 + + +Executor = ExecutorAsyncMultiproc diff --git a/src/fluidimage/executors/exec_async_sequential.py b/src/fluidimage/executors/exec_async_sequential.py --- a/src/fluidimage/executors/exec_async_sequential.py +++ b/src/fluidimage/executors/exec_async_sequential.py @@ -76,3 +76,6 @@ if work.output_queue is not None: work.output_queue[key] = ret self.nb_working_workers_cpu -= 1 + + +Executor = ExecutorAsyncSequential diff --git a/src/fluidimage/executors/exec_async_servers.py b/src/fluidimage/executors/exec_async_servers.py --- a/src/fluidimage/executors/exec_async_servers.py +++ b/src/fluidimage/executors/exec_async_servers.py @@ -267,7 +267,4 @@ return worker -class ExecutorAsyncServersThreading(ExecutorAsyncServers): - """Just used to get a better coverage""" - - _type_server = "threading" +Executor = ExecutorAsyncServers diff --git a/src/fluidimage/executors/exec_async_servers_threading.py b/src/fluidimage/executors/exec_async_servers_threading.py new file mode 100644 --- /dev/null +++ b/src/fluidimage/executors/exec_async_servers_threading.py @@ -0,0 +1,10 @@ +from .exec_async_servers import ExecutorAsyncServers + + +class ExecutorAsyncServersThreading(ExecutorAsyncServers): + """Just used to get a better coverage""" + + _type_server = "threading" + + +Executor = ExecutorAsyncServersThreading diff --git a/src/fluidimage/executors/exec_sequential.py b/src/fluidimage/executors/exec_sequential.py --- a/src/fluidimage/executors/exec_sequential.py +++ b/src/fluidimage/executors/exec_sequential.py @@ -65,3 +65,6 @@ if work.output_queue is not None: work.output_queue[key] = ret + + +Executor = ExecutorSequential diff --git a/src/fluidimage/executors/meson.build b/src/fluidimage/executors/meson.build --- a/src/fluidimage/executors/meson.build +++ b/src/fluidimage/executors/meson.build @@ -6,6 +6,7 @@ 'exec_async_multiproc.py', 'exec_async_sequential.py', 'exec_async_servers.py', + 'exec_async_servers_threading.py', 'exec_sequential.py', 'multi_exec_async.py', 'servers.py', diff --git a/src/fluidimage/executors/multi_exec_async.py b/src/fluidimage/executors/multi_exec_async.py --- a/src/fluidimage/executors/multi_exec_async.py +++ b/src/fluidimage/executors/multi_exec_async.py @@ -335,3 +335,6 @@ for process in self.processes: process.join() + + +Executor = MultiExecutorAsync diff --git a/src/fluidimage/topologies/base.py b/src/fluidimage/topologies/base.py --- a/src/fluidimage/topologies/base.py +++ b/src/fluidimage/topologies/base.py @@ -20,7 +20,7 @@ from fluidimage.util import cstring, logger -from ..executors import ExecutorBase, executors +from ..executors import ExecutorBase, import_executor_class, get_executor_names class Work: @@ -195,13 +195,13 @@ executor = "multi_exec_async" if not isinstance(executor, ExecutorBase): - if executor not in executors: + if executor not in get_executor_names(): raise NotImplementedError(f"executor {executor} does not exist") if nb_max_workers is None: nb_max_workers = self.nb_max_workers - exec_class = executors[executor] + exec_class = import_executor_class(executor) self.executor = exec_class( self, path_dir_result=self.path_dir_result, # HG changeset patch # User paugier <pierre.augier@univ-grenoble-alpes.fr> # Date 1708434964 -3600 # Tue Feb 20 14:16:04 2024 +0100 # Node ID a071b8f85e370455daecebeb42446b68b07de9f1 # Parent b3decb86cc35af45c43859393575bbf73f8ffa6f transonic-clean-dir diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -13,9 +13,7 @@ rm -rf build cleantransonic: - find src -type d -name __pythran__ | xargs rm -rf - find src -type d -name __python__ | xargs rm -rf - find src -type d -name __numba__ | xargs rm -rf + pdm run transonic-clean-dir src cleanall: clean cleantransonic diff --git a/pdm.lock b/pdm.lock --- a/pdm.lock +++ b/pdm.lock @@ -33,7 +33,7 @@ [[package]] name = "anyio" -version = "4.2.0" +version = "4.3.0" requires_python = ">=3.8" summary = "High level compatibility layer for multiple asynchronous event loop implementations" groups = ["dev-doc"] @@ -44,8 +44,8 @@ "typing-extensions>=4.1; python_version < \"3.11\"", ] files = [ - {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, - {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, + {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"}, + {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"}, ] [[package]] @@ -1211,12 +1211,13 @@ [[package]] name = "json5" -version = "0.9.14" +version = "0.9.17" +requires_python = ">=3.8" summary = "A Python implementation of the JSON5 data format." groups = ["dev-doc"] files = [ - {file = "json5-0.9.14-py2.py3-none-any.whl", hash = "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f"}, - {file = "json5-0.9.14.tar.gz", hash = "sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02"}, + {file = "json5-0.9.17-py2.py3-none-any.whl", hash = "sha256:f8ec1ecf985951d70f780f6f877c4baca6a47b6e61e02c4cd190138d10a7805a"}, + {file = "json5-0.9.17.tar.gz", hash = "sha256:717d99d657fa71b7094877b1d921b1cce40ab444389f6d770302563bb7dfd9ae"}, ] [[package]] @@ -1424,7 +1425,7 @@ [[package]] name = "jupyterlab" -version = "4.1.1" +version = "4.1.2" requires_python = ">=3.8" summary = "JupyterLab computational environment" groups = ["dev-doc"] @@ -1445,8 +1446,8 @@ "traitlets", ] files = [ - {file = "jupyterlab-4.1.1-py3-none-any.whl", hash = "sha256:fa3e8c18b804eac04e51ceebd9dd3dd396e08106816f0d09cc426799d7087632"}, - {file = "jupyterlab-4.1.1.tar.gz", hash = "sha256:8acc9f561729d8f32c14c294c397917cddfeeb13a5d46f811979b71b4911a9fd"}, + {file = "jupyterlab-4.1.2-py3-none-any.whl", hash = "sha256:aa88193f03cf4d3555f6712f04d74112b5eb85edd7d222c588c7603a26d33c5b"}, + {file = "jupyterlab-4.1.2.tar.gz", hash = "sha256:5d6348b3ed4085181499f621b7dfb6eb0b1f57f3586857aadfc8e3bf4c4885f9"}, ] [[package]] @@ -1960,9 +1961,9 @@ [[package]] name = "nbconvert" -version = "7.16.0" +version = "7.16.1" requires_python = ">=3.8" -summary = "Converting Jupyter Notebooks" +summary = "Converting Jupyter Notebooks (.ipynb files) to other formats. Output formats include asciidoc, html, latex, markdown, pdf, py, rst, script. nbconvert can be used both as a Python library (`import nbconvert`) or as a command line tool (invoked as `jupyter nbconvert ...`)." groups = ["dev-doc"] dependencies = [ "beautifulsoup4", @@ -1983,8 +1984,8 @@ "traitlets>=5.1", ] files = [ - {file = "nbconvert-7.16.0-py3-none-any.whl", hash = "sha256:ad3dc865ea6e2768d31b7eb6c7ab3be014927216a5ece3ef276748dd809054c7"}, - {file = "nbconvert-7.16.0.tar.gz", hash = "sha256:813e6553796362489ae572e39ba1bff978536192fb518e10826b0e8cadf03ec8"}, + {file = "nbconvert-7.16.1-py3-none-any.whl", hash = "sha256:3188727dffadfdc9c6a1c7250729063d7bc78b355ad7aa023138afa030d1cd07"}, + {file = "nbconvert-7.16.1.tar.gz", hash = "sha256:e79e6a074f49ba3ed29428ed86487bf51509d9aab613bd8522ac08f6d28fd7fd"}, ] [[package]] @@ -3547,7 +3548,7 @@ [[package]] name = "transonic" -version = "0.6.2" +version = "0.6.3" requires_python = ">=3.9" summary = "Make your Python code fly at transonic speeds!" groups = ["build", "default", "test"] @@ -3558,8 +3559,8 @@ "numpy", ] files = [ - {file = "transonic-0.6.2-py3-none-any.whl", hash = "sha256:07534c54354276e3d63632b075d6c275ab3a4f3b1009d7a38771e88cb988031e"}, - {file = "transonic-0.6.2.tar.gz", hash = "sha256:74fa7acc5d819a3b8507d01600a17ab18ae7869ec59bf0ef7b1c9a8b92c58a7e"}, + {file = "transonic-0.6.3-py3-none-any.whl", hash = "sha256:49c44686cfaf1a7a0e18f1fc9d286445b54d8b8216d0af77552f1311db99112a"}, + {file = "transonic-0.6.3.tar.gz", hash = "sha256:993edd14b9a3dada4438167a65ff5d4cd6f904c45e4fbc66f1ba34f0fab42e05"}, ] [[package]] # HG changeset patch # User paugier <pierre.augier@univ-grenoble-alpes.fr> # Date 1708435444 -3600 # Tue Feb 20 14:24:04 2024 +0100 # Node ID 584f7970bc8318e805bc63b0020b063db73a75f2 # Parent a071b8f85e370455daecebeb42446b68b07de9f1 Doc entry points diff --git a/src/fluidimage/executors/__init__.py b/src/fluidimage/executors/__init__.py --- a/src/fluidimage/executors/__init__.py +++ b/src/fluidimage/executors/__init__.py @@ -23,6 +23,12 @@ exec_async_servers servers +.. autofunction:: get_entry_points + +.. autofunction:: get_executor_names + +.. autofunction:: import_executor_class + """ import importlib