v0.8.0: Improved logging

.github/workflows/release.yml

@@ -20,4 +20,4 @@ jobs:
       - name: Build SDist and wheel
         run: pipx run build
       - name: Publish package distributions to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1

.readthedocs.yaml

@@ -17,8 +17,7 @@ build:
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:
-   configuration: docs/conf.py
-
+  configuration: docs/conf.py
 # Optionally build your docs in additional formats such as PDF and ePub
 # formats:
 #    - pdf
@@ -29,4 +28,4 @@ sphinx:
 # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 # python:
 #    install:
-#    - requirements: docs/requirements.txt
+#    - requirements: docs/requirements.txt

README.rst

@@ -345,6 +345,7 @@ The project is reasonably easy:
 Changes
 -------
 
+-  0.8.0: Added extensive logging and improved typing.
 -  0.7.0: Warning if a Batch is flushed multiple times, as we noticed this to be a common indentation error.
 -  0.6.2: Fixes recursive distribution guard, which seemed to be broken.
 -  0.6.1: Bugfixes in naming
@@ -382,4 +383,4 @@ Further contributors are Matthias Konitzny and Patrick Blumenberg.
    :target: https://github.com/d-krupke/slurminade
 
 .. |License| image:: https://img.shields.io/github/license/d-krupke/slurminade
-   :target: https://github.com/d-krupke/slurminade
+   :target: https://github.com/d-krupke/slurminade

docs/conf.py

@@ -5,28 +5,33 @@
 
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../src'))
-autodoc_mock_imports = ['simple_slurm']
+
+sys.path.insert(0, os.path.abspath("../src"))
+autodoc_mock_imports = ["simple_slurm"]
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-project = 'slurminade'
-copyright = '2023, Dominik Krupke'
-author = 'Dominik Krupke (TU Braunschweig, IBR, Algorithms Group)'
+project = "slurminade"
+copyright = "2023, Dominik Krupke"
+author = "Dominik Krupke (TU Braunschweig, IBR, Algorithms Group)"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', 'sphinx_rtd_theme']
-
-templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+    "sphinx_rtd_theme",
+]
 
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'sphinx_rtd_theme'
-html_static_path = ['_static']
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]

docs/slurminade.rst

@@ -75,5 +75,3 @@ slurminade.options module
    :members:
    :undoc-members:
    :show-inheritance:
-
-

examples/example_2.py

@@ -1,10 +1,10 @@
 import datetime
 
-import slurminade
-
 # example with importing a function to be executed by node
 from example_2b import f
 
+import slurminade
+
 slurminade.set_dispatch_limit(2)
 slurminade.update_default_configuration(partition="alg", constraint="alggen02")
 

src/slurminade/__init__.py

@@ -11,15 +11,20 @@
     # compatible with any environment.
     # You can enforce slurm with `slurminade.set_dispatcher(slurminade.SlurmDispatcher())`
 
+
     # use this decorator to make a function distributable with slurm
-    @slurminade.slurmify(constraint="alggen02")  # function specific options can be specified
+    @slurminade.slurmify(
+        constraint="alggen02"
+    )  # function specific options can be specified
     def prepare():
         print("Prepare")
 
+
     @slurminade.slurmify()
     def f(foobar):
         print(f"f({foobar})")
 
+
     @slurminade.slurmify()
     def clean_up():
         print("Clean up")
@@ -52,7 +57,11 @@ def clean_up():
 # flake8: noqa F401
 from .function import slurmify
 from .conf import update_default_configuration, set_default_configuration
-from .guard import set_dispatch_limit, allow_recursive_distribution, disable_warning_on_repeated_flushes
+from .guard import (
+    set_dispatch_limit,
+    allow_recursive_distribution,
+    disable_warning_on_repeated_flushes,
+)
 from .batch import Batch
 from .dispatcher import (
     srun,
@@ -80,4 +89,11 @@ def clean_up():
     "get_dispatcher",
     "TestDispatcher",
     "SubprocessDispatcher",
-]
+    "set_entry_point",
+]
+
+# set default logging
+import logging
+
+logging.getLogger("slurminade").setLevel(logging.INFO)
+logging.getLogger("slurminade").addHandler(logging.StreamHandler())

src/slurminade/batch.py

@@ -1,6 +1,7 @@
 """
 Contains code for bundling function calls together.
 """
+import logging
 import typing
 from collections import defaultdict
 
@@ -11,8 +12,8 @@
     set_dispatcher,
 )
 from .function import SlurmFunction
-from .options import SlurmOptions
 from .guard import BatchGuard
+from .options import SlurmOptions
 
 
 class TaskBuffer:
@@ -64,9 +65,7 @@ def __init__(self, max_size: int):
         self._tasks = TaskBuffer()
         self._batch_guard = BatchGuard()
 
-    def flush(
-        self, options: typing.Optional[SlurmOptions] = None
-    ) -> typing.List[int]:
+    def flush(self, options: typing.Optional[SlurmOptions] = None) -> typing.List[int]:
         """
         Distribute all buffered tasks. Return the job ids used.
         This method is called automatically when the context is exited.
@@ -112,10 +111,20 @@ def _dispatch(
             self._tasks.add(func, options)
         return -1
 
-    def srun(self, command: str, conf: dict = None, simple_slurm_kwargs: dict = None):
+    def srun(
+        self,
+        command: str,
+        conf: typing.Optional[typing.Dict] = None,
+        simple_slurm_kwargs: typing.Optional[typing.Dict] = None,
+    ):
         return self.subdispatcher.srun(command, conf, simple_slurm_kwargs)
 
-    def sbatch(self, command: str, conf: dict = None, simple_slurm_kwargs: dict = None):
+    def sbatch(
+        self,
+        command: str,
+        conf: typing.Optional[typing.Dict] = None,
+        simple_slurm_kwargs: typing.Optional[typing.Dict] = None,
+    ):
         return self.subdispatcher.sbatch(command, conf, simple_slurm_kwargs)
 
     def __enter__(self):
@@ -130,6 +139,16 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.flush()
         set_dispatcher(self.subdispatcher)
 
+    def _log_dispatch(self, funcs: typing.List[FunctionCall], options: SlurmOptions):
+        if len(funcs) == 1:
+            logging.getLogger("slurminade").info(
+                f"Adding task to batch with options {options}: {funcs[0]}"
+            )
+        else:
+            logging.getLogger("slurminade").info(
+                f"Adding {len(funcs)} tasks to batch with options {options}: {', '.join([str(f) for f in funcs])}"
+            )
+
     def __del__(self):
         self.flush()