merge develop

.gitignore

@@ -176,3 +176,5 @@ bar
 *.zarr/
 ~$images.pptx
 test.py
+cutout.png
+*.out

.pre-commit-config.yaml

@@ -10,10 +10,11 @@ repos:
     -   id: debug-statements # Check for debugger imports and py37+ breakpoint()
     -   id: end-of-file-fixer # Ensure files end in a newline
     -   id: trailing-whitespace # Trailing whitespace checker
--   repo: https://github.com/asottile/reorder-python-imports # Reorder imports
-    rev: v3.10.0
-    hooks:
-    -   id: reorder-python-imports
+# conflicting with "isort"
+# -   repo: https://github.com/asottile/reorder-python-imports # Reorder imports
+#     rev: v3.10.0
+#     hooks:
+#     -   id: reorder-python-imports
 -   repo: https://github.com/asottile/pyupgrade # Upgrade Python syntax
     rev: v3.7.0
     hooks:

ecml_tools/__init__.py

@@ -5,4 +5,4 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-__version__ = "0.4.5"
+__version__ = "0.4.6"

ecml_tools/__main__.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+
+import argparse
+import logging
+import sys
+import traceback
+
+from . import __version__
+from .commands import COMMANDS
+
+LOG = logging.getLogger(__name__)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument(
+        "--version",
+        "-V",
+        action="store_true",
+        help="show the version and exit",
+    )
+    parser.add_argument(
+        "--debug",
+        "-d",
+        action="store_true",
+        help="Debug mode",
+    )
+
+    subparsers = parser.add_subparsers(help="commands:", dest="command")
+    for name, command in COMMANDS.items():
+        command_parser = subparsers.add_parser(name, help=command.__doc__)
+        command.add_arguments(command_parser)
+
+    args = parser.parse_args()
+
+    if args.version:
+        print(__version__)
+        return
+
+    if args.command is None:
+        parser.print_help()
+        return
+
+    cmd = COMMANDS[args.command]
+
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=logging.DEBUG if args.debug else logging.INFO,
+    )
+
+    try:
+        cmd.run(args)
+    except ValueError as e:
+        traceback.print_exc()
+        LOG.error("\n💣 %s", str(e).lstrip())
+        LOG.error("💣 Exiting")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

ecml_tools/commands/__init__.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import argparse
+import importlib
+import logging
+import os
+import sys
+
+LOG = logging.getLogger(__name__)
+
+
+def register(here, package, select, fail=None):
+    result = {}
+    not_available = {}
+
+    for p in os.listdir(here):
+        full = os.path.join(here, p)
+        if p.startswith("_"):
+            continue
+        if not (
+            p.endswith(".py")
+            or (
+                os.path.isdir(full)
+                and os.path.exists(os.path.join(full, "__init__.py"))
+            )
+        ):
+            continue
+
+        name, _ = os.path.splitext(p)
+
+        try:
+            imported = importlib.import_module(
+                f".{name}",
+                package=package,
+            )
+        except ImportError as e:
+            not_available[name] = e
+            continue
+
+        obj = select(imported)
+        if obj is not None:
+            result[name] = obj
+
+    for name, e in not_available.items():
+        if fail is None:
+            pass
+        if callable(fail):
+            result[name] = fail(name, e)
+
+    return result
+
+
+class Command:
+    def run(self, args):
+        raise NotImplementedError(f"Command not implemented: {args.command}")
+
+
+class Failed(Command):
+    def __init__(self, name, error):
+        self.name = name
+        self.error = error
+
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("x", nargs=argparse.REMAINDER)
+
+    def run(self, args):
+        print(f"Command '{self.name}' not available: {self.error}")
+        sys.exit(1)
+
+
+COMMANDS = register(
+    os.path.dirname(__file__),
+    __name__,
+    lambda x: x.command(),
+    lambda name, error: Failed(name, error),
+)

ecml_tools/commands/create.py

@@ -0,0 +1,24 @@
+from ecml_tools.create import Creator
+
+from . import Command
+
+
+class Create(Command):
+    internal = True
+    timestamp = True
+
+    def add_arguments(self, command_parser):
+        command_parser.add_argument(
+            "--overwrite", action="store_true", help="Overwrite existing files"
+        )
+        command_parser.add_argument("config", help="Configuration file")
+        command_parser.add_argument("path", help="Path to store the created data")
+
+    def run(self, args):
+        kwargs = vars(args)
+
+        c = Creator(**kwargs)
+        c.create()
+
+
+command = Create

ecml_tools/commands/inspect/__init__.py

@@ -0,0 +1,45 @@
+# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import os
+
+from .. import Command
+
+# from .checkpoint import InspectCheckpoint
+from .zarr import InspectZarr
+
+
+class Inspect(Command, InspectZarr):
+    # class Inspect(Command, InspectCheckpoint, InspectZarr):
+    """Inspect a checkpoint or zarr file."""
+
+    def add_arguments(self, command_parser):
+        # g = command_parser.add_mutually_exclusive_group()
+        # g.add_argument("--inspect", action="store_true", help="Inspect weights")
+        command_parser.add_argument("path", metavar="PATH", nargs="+")
+        command_parser.add_argument("--detailed", action="store_true")
+        # command_parser.add_argument("--probe", action="store_true")
+        command_parser.add_argument("--progress", action="store_true")
+        command_parser.add_argument("--statistics", action="store_true")
+        command_parser.add_argument("--size", action="store_true", help="Print size")
+
+    def run(self, args):
+        dic = vars(args)
+        for path in dic.pop("path"):
+            if (
+                os.path.isdir(path)
+                or path.endswith(".zarr.zip")
+                or path.endswith(".zarr")
+            ):
+                self.inspect_zarr(path=path, **dic)
+            else:
+                raise ValueError(f"Unknown file type: {path}")
+                # self.inspect_checkpoint(path=path, **dic)
+
+
+command = Inspect