Skip to content

Commit

Permalink
multi_exec_subproc_sync and doc new executors
Browse files Browse the repository at this point in the history
  • Loading branch information
paugier committed May 24, 2024
1 parent ec042f0 commit 7fe47a8
Show file tree
Hide file tree
Showing 8 changed files with 50 additions and 4 deletions.
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,11 @@ exec_sequential = "fluidimage.executors.exec_sequential"
exec_async = "fluidimage.executors.exec_async"
exec_async_sequential = "fluidimage.executors.exec_async_sequential"
exec_async_seq_for_multi = "fluidimage.executors.exec_async_seq_for_multi"
exec_seq_for_multi = "fluidimage.executors.exec_seq_for_multi"
multi_exec_async = "fluidimage.executors.multi_exec_async"
multi_exec_sync = "fluidimage.executors.multi_exec_sync"
multi_exec_subproc = "fluidimage.executors.multi_exec_subproc"
multi_exec_subproc_sync = "fluidimage.executors.multi_exec_subproc_sync"
exec_async_multi = "fluidimage.executors.exec_async_multiproc"
exec_async_servers = "fluidimage.executors.exec_async_servers"
exec_async_servers_threading = "fluidimage.executors.exec_async_servers_threading"
Expand Down
5 changes: 4 additions & 1 deletion src/fluidimage/executors/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@
exec_async
exec_async_sequential
multi_exec_async
multi_exec_sync
multi_exec_subproc
multi_exec_subproc_sync
exec_async_seq_for_multi
exec_seq_for_multi
exec_async_multiproc
exec_async_servers
servers
Expand Down Expand Up @@ -58,7 +61,7 @@ def afterfork():
# because the OS does not support forks (or not fully) and
# multiprocessing works differently than on Linux
# see https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
supported_multi_executors = ["multi_exec_subproc"]
supported_multi_executors = ["multi_exec_subproc", "multi_exec_subproc_sync"]
if sys.platform == "linux":
supported_multi_executors.insert(0, "multi_exec_sync")
supported_multi_executors.insert(0, "multi_exec_async")
Expand Down
11 changes: 11 additions & 0 deletions src/fluidimage/executors/exec_seq_for_multi.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
"""Sequential executor for multi executor
.. autoclass:: ExecutorSeqForMulti
:members:
:private-members:
"""

from fluidimage.executors.exec_async_seq_for_multi import ExecutorAsyncSeqForMulti
from fluidimage.executors.exec_sequential import ExecutorSequential


class ExecutorSeqForMulti(ExecutorSequential, ExecutorAsyncSeqForMulti):
"""Sequential executor modified for multi executors"""


Executor = ExecutorSeqForMulti
1 change: 1 addition & 0 deletions src/fluidimage/executors/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ python_sources = [
'multi_exec_async.py',
'multi_exec_sync.py',
'multi_exec_subproc.py',
'multi_exec_subproc_sync.py',
'servers.py',
]

Expand Down
7 changes: 4 additions & 3 deletions src/fluidimage/executors/multi_exec_subproc.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@ class MultiExecutorSubproc(MultiExecutorBase):
"""Multi executor based on subprocesses and splitters"""

splitter: Splitter
executor_for_multi = "exec_async_seq_for_multi"

def _init_num_expected_results(self):

try:
splitter_cls = self.topology.Splitter
except AttributeError as error:
raise ValueError(
"MultiExecutorSubproc can only execute "
f"{type(self).__name__} can only execute "
"topologies with a Splitter."
) from error

Expand All @@ -39,12 +40,12 @@ def _init_num_expected_results(self):
params._set_child(
"compute_kwargs",
attribs={
"executor": "exec_async_seq_for_multi",
"executor": self.executor_for_multi,
"nb_max_workers": 1,
},
)
except ValueError:
params.compute_kwargs.executor = "exec_async_seq_for_multi"
params.compute_kwargs.executor = self.executor_for_multi
params.compute_kwargs.nb_max_workers = 1

try:
Expand Down
16 changes: 16 additions & 0 deletions src/fluidimage/executors/multi_exec_subproc_sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""Multi executor based on subprocesses using exec_seq_for_multi
.. autoclass:: MultiExecutorSubprocSync
:members:
:private-members:
"""

from fluidimage.executors.multi_exec_subproc import MultiExecutorSubproc


class MultiExecutorSubprocSync(MultiExecutorSubproc):
executor_for_multi = "exec_seq_for_multi"


Executor = MultiExecutorSubprocSync
10 changes: 10 additions & 0 deletions src/fluidimage/executors/multi_exec_sync.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
"""
Multi executors sync
=====================
.. autoclass:: MultiExecutorSync
:members:
:private-members:
"""

from fluidimage.executors.exec_seq_for_multi import ExecutorSeqForMulti
from fluidimage.executors.multi_exec_async import MultiExecutorAsync

Expand Down
2 changes: 2 additions & 0 deletions src/fluidimage/topologies/test_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@

# tmp: TopologyExample doesn't have a Splitter
executors.remove("multi_exec_subproc")
executors.remove("multi_exec_subproc_sync")


@pytest.mark.usefixtures("close_plt_figs")
@pytest.mark.parametrize("executor", executors)
def test_topo_example(tmp_path_karman, executor):

Expand Down

0 comments on commit 7fe47a8

Please sign in to comment.