Merge pull request #139 from eyecan-ai/develop

Develop

pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "info@eyecan.ai"
-__version__ = "1.8.0"
+__version__ = "1.8.1"

pipelime/choixe/utils/imports.py

@@ -286,12 +286,14 @@ def import_symbol(
                         symbol_name.insert(0, cl_path)
 
                 if module_ is None:
-                    raise ModuleNotFoundError("Module not found")
+                    raise ModuleNotFoundError(
+                        f"Invalid symbol class path `{symbol_definition}`"
+                    )
             else:
-                raise ImportError("unknow definition")
+                raise ValueError(f"Unsupported symbol definition `{symbol_definition}`")
 
         if not symbol_name:
-            raise ImportError("No symbol name found")
+            raise ValueError(f"No symbol name found in `{symbol_definition}`")
 
         # import the symbol
         # if nested, we need to import the parent symbols first

pipelime/cli/utils.py

@@ -1,5 +1,7 @@
 import typing as t
+from pathlib import Path
 from types import ModuleType
+from loguru import logger
 
 from pydantic import BaseModel, ValidationError
 
@@ -58,14 +60,17 @@ def _complete(incomplete: str):
 
     @classmethod
     def set_extra_modules(cls, modules: t.Sequence[str]):
-        cls.extra_modules = list(modules)
+        cls.extra_modules = [cls._normalize_module_path(m) for m in modules]
 
     @classmethod
     def register_extra_module(cls, module: str):
         """Register an extra module to be loaded when importing everything.
         Mainly used for dynamically imported symbols.
         """
-        if module not in cls.extra_modules:
+        module = cls._normalize_module_path(module)
+        if (
+            module.endswith(".py") or not module.startswith("pipelime")
+        ) and module not in cls.extra_modules:
             cls.extra_modules.append(module)
 
     @classmethod
@@ -80,6 +85,12 @@ def register_action(cls, name: str, info: ActionInfo):
             raise ValueError(f"Action `{name}` already registered")
         cls.registered_actions[name] = info
 
+    @classmethod
+    def _normalize_module_path(cls, module: str):
+        if module.endswith(".py"):
+            return Path(module).resolve().absolute().as_posix()
+        return module
+
     @classmethod
     def _symbol_name(cls, symbol):
         from pydantic import BaseModel
@@ -205,6 +216,7 @@ def get_symbol(
         symbol_path: str,
         base_cls: t.Type,
         symbol_cache: t.Mapping[t.Tuple[str, str], t.Mapping],
+        raise_if_not_found: bool = False,
     ):
         import pipelime.choixe.utils.imports as pl_imports
 
@@ -220,24 +232,30 @@ def get_symbol(
                     imported_symbol._classpath = symbol_path
 
                 return imported_symbol
-            except (ImportError, TypeError):
+            except Exception as e:
+                if raise_if_not_found:
+                    raise e
+                logger.warning(f"Could not import {symbol_path}: {e}")
                 return None
 
         for sym_type, sym_dict in symbol_cache.items():
             if symbol_path in sym_dict:
                 return (sym_type, sym_dict[symbol_path])
+        if raise_if_not_found:
+            raise ValueError(f"{symbol_path} not found")
         return None
 
     @classmethod
-    def get_command(cls, command_name: str):
+    def get_command(cls, command_name: str, raise_if_not_found: bool = False):
         from pipelime.piper import PipelimeCommand
 
         sym_cls = cls.get_symbol(
-            command_name, PipelimeCommand, cls.get_pipelime_commands()
+            command_name,
+            PipelimeCommand,
+            cls.get_pipelime_commands(),
+            raise_if_not_found,
         )
-        if sym_cls is None:
-            return None
-        if not isinstance(sym_cls, tuple):
+        if sym_cls is not None and not isinstance(sym_cls, tuple):
             return (("Imported Command", "Imported Commands"), sym_cls)
         return sym_cls
 
@@ -249,13 +267,13 @@ def get_operator(cls, operator_name: str):
         return None
 
     @classmethod
-    def get_stage(cls, stage_name: str):
+    def get_stage(cls, stage_name: str, raise_if_not_found: bool = False):
         from pipelime.stages import SampleStage
 
-        sym_cls = cls.get_symbol(stage_name, SampleStage, cls.get_sample_stages())
-        if sym_cls is None:
-            return None
-        if not isinstance(sym_cls, tuple):
+        sym_cls = cls.get_symbol(
+            stage_name, SampleStage, cls.get_sample_stages(), raise_if_not_found
+        )
+        if sym_cls is not None and not isinstance(sym_cls, tuple):
             return (("Imported Stage", "Imported Stages"), sym_cls)
         return sym_cls
 

pipelime/commands/piper.py

@@ -551,7 +551,13 @@ def draw_graph(self, output: t.Optional[Path] = None, **kwargs) -> None:
         if "o" not in kwargs and "output" not in kwargs:
             kwargs["output"] = output
         drawer = DrawCommand(
-            nodes=nodes, include=self.include, exclude=self.exclude, **kwargs  # type: ignore
+            nodes=nodes,
+            include=self.include,
+            exclude=self.exclude,
+            skip_on_error=self.skip_on_error,
+            start_from=self.start_from,
+            stop_at=self.stop_at,
+            **kwargs,
         )
         drawer.run()
 

pipelime/piper/model.py

@@ -506,31 +506,27 @@ def create(
 
         if isinstance(value, NodesDefinition):
             return value
-        try:
-            plnodes = {}
-            for name, cmd in value.items():
-                if checkpoint:
-                    ckpt = checkpoint.get_namespace(
-                        name.replace(".", "_").replace("[", "_").replace("]", "_")
-                    )
-                else:
-                    ckpt = None
-
-                try:
-                    plcmd = get_pipelime_command(cmd, ckpt)
-                except ValidationError:
-                    if skip_on_error:
-                        logger.warning(
-                            f"Skipping node `{name}` due to validation error."
-                        )
-                    else:
-                        raise
+
+        plnodes = {}
+        for name, cmd in value.items():
+            if checkpoint:
+                ckpt = checkpoint.get_namespace(
+                    name.replace(".", "_").replace("[", "_").replace("]", "_")
+                )
+            else:
+                ckpt = None
+
+            try:
+                plcmd = get_pipelime_command(cmd, ckpt)
+            except ValidationError as e:
+                if skip_on_error:
+                    logger.warning(f"Skipping node `{name}` due to validation error.")
                 else:
-                    plnodes[name] = plcmd
-            return cls(__root__=plnodes)
-        except ValidationError as e:
-            show_field_alias_valerr(e)
-            raise e
+                    show_field_alias_valerr(e)
+                    raise ValueError(f"Invalid node definition `{name}`") from e
+            else:
+                plnodes[name] = plcmd
+        return cls(__root__=plnodes)
 
     @property
     def value(self):

pipelime/piper/progress/listener/callbacks/file.py

@@ -44,6 +44,7 @@ def __init__(
             self._dump = json.dump
             self._load = json.load
 
+        self._data = {}
         self._filename.touch()
 
         logger.info(f"Writing progress to {filename}")

pipelime/sequences/samples_sequence.py

@@ -356,7 +356,7 @@ def from_images(
         *,
         must_exist: bool = True,
         image_key: str = "image",
-        sort_files: bool = True,
+        sort_files: bool = False,
         recursive: bool = True,
     ) -> SamplesSequence:
         """A SamplesSequence loading images from a folder.

pipelime/utils/inspection.py

@@ -1,17 +1,15 @@
 from typing import Optional
 
 
-class MyCaller:
-    def __init__(self, steps_back: int = 0):
-        """Get information about your caller.
-
-        Args:
-            steps_back (int, optional): use 1 to get the caller of your caller, 2 to get
-                the grandparent of your caller, etc. Defaults to 0.
-        """
-        import inspect
+class _StackFrameInfo:
+    def __init__(self, stack, pos):
+        self._stack = stack
+        self._pos = pos
+        self._caller = stack[pos]
 
-        self._caller = inspect.stack()[2 + steps_back]
+    @property
+    def parent(self):
+        return _StackFrameInfo(self._stack, self._pos + 1)
 
     @property
     def globals(self):
@@ -25,6 +23,14 @@ def locals(self):
     def filename(self) -> str:
         return self._caller.filename
 
+    @property
+    def lineno(self) -> int:
+        return self._caller.lineno
+
+    @property
+    def function_name(self) -> str:
+        return self._caller.function
+
     @property
     def module(self) -> str:
         return self._caller.frame.f_globals["__name__"]
@@ -38,3 +44,15 @@ def docstrings(self) -> Optional[str]:
         import inspect
 
         return inspect.getdoc(self._caller.frame.f_globals.get(self._caller.function))
+
+
+def MyCaller(steps_back: int = 0):
+    """Get information about your caller.
+
+    Args:
+        steps_back (int, optional): use 1 to get the caller of your caller, 2 to get
+            the grandparent of your caller, etc. Defaults to 0.
+    """
+    import inspect
+
+    return _StackFrameInfo(inspect.stack(), 2 + steps_back)

tests/conftest.py

@@ -107,8 +107,6 @@ def choixe_plain_cfg(choixe_folder: Path) -> Path:
 
 @pytest.fixture(scope="function")
 def all_dags(piper_folder: Path) -> t.Sequence[t.Mapping[str, t.Any]]:
-    import pipelime.choixe.utils.io as choixe_io  # noqa: F401
-    from pipelime.choixe import XConfig  # noqa: F401
     from . import TestUtils
 
     def _add_if_exists(out, path, key):

tests/pipelime/cli/test_base.py

@@ -31,7 +31,25 @@ def test_help(self, extra_modules):
                 )
                 assert cmd in result.output
 
-    def test_list(self, extra_modules):
+    def test_list(self):
+        result = self._base_launch(["list"])
+        assert "Pipelime Commands" in result.output
+        assert "Sample Stages" in result.output
+        assert "Sequence Generators" in result.output
+        assert "Sequence Piped Operations" in result.output
+
+        # NB: the names are different
+        result = self._base_launch(["--verbose", "list"])
+        assert "Pipelime Command" in result.output
+        assert "Sample Stage" in result.output
+        assert "Sequence Generator" in result.output
+        assert "Sequence Piped Operation" in result.output
+        assert "Fields" in result.output
+        assert "Description" in result.output
+        assert "Type" in result.output
+        assert "Default" in result.output
+
+    def test_list_extra(self, extra_modules):
         def _check(result, *, in_modules=[], not_in_modules=[]):
             for v in in_modules:
                 assert v in result.output

tests/pipelime/piper/test_commands.py

@@ -98,19 +98,40 @@ def test_draw(self, all_dags: t.Sequence[t.Mapping[str, t.Any]], tmp_path: Path)
                     assert g_out is not None
                     assert g_ref == g_out
 
-    def test_run_dag(self, all_dags: t.Sequence[t.Mapping[str, t.Any]]):
+    @pytest.mark.parametrize(
+        "watch", [True, False, "rich", "tqdm", Path("cmdout.json"), None]
+    )
+    def test_run_dag(
+        self,
+        all_dags: t.Sequence[t.Mapping[str, t.Any]],
+        watch: t.Union[bool, str, Path, None],
+        tmp_path: Path,
+    ):
         from pipelime.commands.piper import RunCommand
+        import shutil
+        from pipelime.choixe.utils.io import PipelimeTmp
+
+        if isinstance(watch, Path):
+            watch = tmp_path / watch
 
         for dag in all_dags:
-            cmd = RunCommand(**(dag["config"]))
+            cmd = RunCommand(watch=watch, **(dag["config"]))  # type: ignore
             cmd()
 
+            if isinstance(watch, Path):
+                assert watch.exists()
+
             # output folders now exist, so the commands should fail
             # when creating the output pipes
             cmd = RunCommand(**(dag["config"]))
             with pytest.raises(ValueError):
                 cmd()
 
+            if PipelimeTmp.SESSION_TMP_DIR is not None:
+                for path in PipelimeTmp.SESSION_TMP_DIR.iterdir():
+                    if path.is_dir():
+                        shutil.rmtree(path, ignore_errors=True)
+
     def test_port_forwarding(self, piper_folder: Path, tmp_path: Path):
         import pipelime.choixe.utils.io as choixe_io
         from pipelime.choixe import XConfig

tests/pipelime/piper/test_dag.py

@@ -269,7 +269,7 @@ def test_skip_on_error(self, minimnist_dataset: dict, decorated: bool):
             debug_folder=None,
             skip_on_error=False,
         )
-        with pytest.raises(ValidationError):
+        with pytest.raises(ValueError):
             _ = dag.piper_graph
 
     @pytest.mark.parametrize("decorated", [True, False])