apply qa :

apply black with line lenght 120
apply isort with --force-single-line-imports
comply with ruff
update precomit

.pre-commit-config.yaml

@@ -1,29 +1,47 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-    -   id: check-yaml # Check YAML files for syntax errors
-    -   id: debug-statements # Check for debugger imports and py37+ breakpoint()
-#    -   id: end-of-file-fixer # Ensure files end in a newline
-#    -   id: trailing-whitespace # Trailing whitespace checker
-    -   id: no-commit-to-branch # Prevent committing to main / master
--   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.1.1
-    hooks:
-      - id: black
-        args: [--line-length=120]
--   repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        name: isort (python)
-      - id: isort
-        name: isort (cython)
-        types: [cython]
-      - id: isort
-        name: isort (pyi)
-        types: [pyi]
--   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
-    hooks:
-    -   id: flake8
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
+#  - id: end-of-file-fixer # Ensure files end in a newline
+#  - id: trailing-whitespace # Trailing whitespace checker
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-added-large-files # Check for large files added to git
+  - id: check-merge-conflict # Check for files that contain merge conflict
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.1.1
+  hooks:
+    - id: black
+      args: [--line-length=120]
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
+#- repo: https://github.com/pycqa/flake8
+#  rev: 7.0.0
+#  hooks:
+#  -   id: flake8
+#  #    args: [--exit-zero]
+#  #    verbose: true
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.3.0
+  hooks:
+  - id: ruff
+    args:
+    - --line-length=120
+    - --ignore=E203
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview 
+#- repo: https://github.com/pre-commit/mirrors-mypy
+#  rev: v0.720
+#  hooks:
+#  -   id: mypy
+#      verbose: true
+#      entry: bash -c 'mypy "$@" || true' --

ecml_tools/__main__.py

@@ -21,9 +21,7 @@
 
 
 def main():
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
 
     parser.add_argument(
         "--version",

ecml_tools/commands/__init__.py

@@ -25,13 +25,7 @@ def register(here, package, select, fail=None):
         full = os.path.join(here, p)
         if p.startswith("_"):
             continue
-        if not (
-            p.endswith(".py")
-            or (
-                os.path.isdir(full)
-                and os.path.exists(os.path.join(full, "__init__.py"))
-            )
-        ):
+        if not (p.endswith(".py") or (os.path.isdir(full) and os.path.exists(os.path.join(full, "__init__.py")))):
             continue
 
         name, _ = os.path.splitext(p)

ecml_tools/commands/copy.py

@@ -8,7 +8,8 @@
 import logging
 import os
 import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
 
 import tqdm
 

ecml_tools/commands/create.py

@@ -8,9 +8,7 @@ class Create(Command):
     timestamp = True
 
     def add_arguments(self, command_parser):
-        command_parser.add_argument(
-            "--overwrite", action="store_true", help="Overwrite existing files"
-        )
+        command_parser.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
         command_parser.add_argument("config", help="Configuration file")
         command_parser.add_argument("path", help="Path to store the created data")
 

ecml_tools/commands/inspect/__init__.py

@@ -31,11 +31,7 @@ def add_arguments(self, command_parser):
     def run(self, args):
         dic = vars(args)
         for path in dic.pop("path"):
-            if (
-                os.path.isdir(path)
-                or path.endswith(".zarr.zip")
-                or path.endswith(".zarr")
-            ):
+            if os.path.isdir(path) or path.endswith(".zarr.zip") or path.endswith(".zarr"):
                 self.inspect_zarr(path=path, **dic)
             else:
                 raise ValueError(f"Unknown file type: {path}")

ecml_tools/commands/inspect/zarr.py

@@ -16,9 +16,14 @@
 import semantic_version
 import tqdm
 
-from ecml_tools.data import open_dataset, open_zarr
-from ecml_tools.utils.humanize import bytes, number, when
-from ecml_tools.utils.text import dotted_line, progress, table
+from ecml_tools.data import open_dataset
+from ecml_tools.data import open_zarr
+from ecml_tools.utils.humanize import bytes
+from ecml_tools.utils.humanize import number
+from ecml_tools.utils.humanize import when
+from ecml_tools.utils.text import dotted_line
+from ecml_tools.utils.text import progress
+from ecml_tools.utils.text import table
 
 LOG = logging.getLogger(__name__)
 
@@ -28,9 +33,7 @@ def compute_directory_size(path):
         return None, None
     size = 0
     n = 0
-    for dirpath, _, filenames in tqdm.tqdm(
-        os.walk(path), desc="Computing size", leave=False
-    ):
+    for dirpath, _, filenames in tqdm.tqdm(os.walk(path), desc="Computing size", leave=False):
         for filename in filenames:
             file_path = os.path.join(dirpath, filename)
             size += os.path.getsize(file_path)
@@ -311,21 +314,15 @@ def progress(self):
         assert build_flags.size == build_lengths.size
 
         latest_write_timestamp = self.zarr.attrs.get("latest_write_timestamp")
-        latest = (
-            datetime.datetime.fromisoformat(latest_write_timestamp)
-            if latest_write_timestamp
-            else None
-        )
+        latest = datetime.datetime.fromisoformat(latest_write_timestamp) if latest_write_timestamp else None
 
         if not all(build_flags):
             if latest:
                 print(f"🪫  Dataset not ready, last update {when(latest)}.")
             else:
                 print("🪫  Dataset not ready.")
             total = sum(build_lengths)
-            built = sum(
-                ln if flag else 0 for ln, flag in zip(build_lengths, build_flags)
-            )
+            built = sum(ln if flag else 0 for ln, flag in zip(build_lengths, build_flags))
             print(
                 "📈 Progress:",
                 progress(built, total, width=50),
@@ -411,9 +408,7 @@ def last_date(self):
         assert isinstance(time, int), (time, type(time))
         if time > 100:
             time = time // 100
-        return datetime.datetime.fromisoformat(monthly["stop"]) + datetime.timedelta(
-            hours=time
-        )
+        return datetime.datetime.fromisoformat(monthly["stop"]) + datetime.timedelta(hours=time)
 
     @property
     def frequency(self):
@@ -472,12 +467,8 @@ def _info(self, verbose, history, statistics, **kwargs):
 
         # for backward compatibility
         if "climetlab" in z.attrs:
-            climetlab_version = (
-                z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
-            )
-            print(
-                f"climetlab version used to create this zarr: {climetlab_version}. Not supported."
-            )
+            climetlab_version = z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
+            print(f"climetlab version used to create this zarr: {climetlab_version}. Not supported.")
             return
 
         version = z.attrs.get("version")
@@ -503,12 +494,7 @@ def initialised(self):
                 return datetime.datetime.fromisoformat(record["timestamp"])
 
         # Sometimes the first record is missing
-        timestamps = sorted(
-            [
-                datetime.datetime.fromisoformat(d["timestamp"])
-                for d in self.metadata.get("history", [])
-            ]
-        )
+        timestamps = sorted([datetime.datetime.fromisoformat(d["timestamp"]) for d in self.metadata.get("history", [])])
         if timestamps:
             return timestamps[0]
 

ecml_tools/commands/scan.py

@@ -16,9 +16,7 @@ class Scan(Command):
     timestamp = True
 
     def add_arguments(self, command_parser):
-        command_parser.add_argument(
-            "--extension", default=".grib", help="Extension of the files to scan"
-        )
+        command_parser.add_argument("--extension", default=".grib", help="Extension of the files to scan")
         command_parser.add_argument(
             "--magic",
             help="File 'magic' to use to identify the file type. Overrides --extension",

ecml_tools/create/check.py

@@ -153,7 +153,7 @@ class StatisticsValueError(ValueError):
 
 def check_data_values(arr, *, name: str, log=[], allow_nan=False):
     if allow_nan is False:
-        allow_nan = lambda x: False
+        allow_nan = lambda x: False  # noqa: E731
     if allow_nan(name):
         arr = arr[~np.isnan(arr)]
 

ecml_tools/create/functions/actions/grib.py

@@ -19,9 +19,7 @@ def check(ds, paths, **kwargs):
             count *= len(v)
 
     if len(ds) != count:
-        raise ValueError(
-            f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})"
-        )
+        raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})")
 
 
 def execute(context, dates, path, *args, **kwargs):
@@ -31,9 +29,7 @@ def execute(context, dates, path, *args, **kwargs):
     dates = [d.isoformat() for d in dates]
 
     for path in given_paths:
-        paths = Pattern(path, ignore_missing_keys=True).substitute(
-            *args, date=dates, **kwargs
-        )
+        paths = Pattern(path, ignore_missing_keys=True).substitute(*args, date=dates, **kwargs)
 
         for name in ("grid", "area", "rotation", "frame", "resol", "bitmap"):
             if name in kwargs:

ecml_tools/create/functions/actions/netcdf.py

@@ -18,9 +18,7 @@ def check(what, ds, paths, **kwargs):
             count *= len(v)
 
     if len(ds) != count:
-        raise ValueError(
-            f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, {what}s={paths})"
-        )
+        raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, {what}s={paths})")
 
 
 def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
@@ -30,9 +28,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
     ds = load_source("empty")
 
     for path in given_paths:
-        paths = Pattern(path, ignore_missing_keys=True).substitute(
-            *args, date=dates, **kwargs
-        )
+        paths = Pattern(path, ignore_missing_keys=True).substitute(*args, date=dates, **kwargs)
 
         levels = kwargs.get("level", kwargs.get("levelist"))
 

ecml_tools/create/functions/actions/source.py

@@ -41,9 +41,7 @@ def source(context, dates, **kwargs):
       time: $from_dates
     """
     )
-    dates = yaml.safe_load(
-        "[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]"
-    )
+    dates = yaml.safe_load("[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]")
     dates = to_datetime_list(dates)
 
     DEBUG = True

ecml_tools/create/functions/actions/tendencies.py

@@ -98,9 +98,7 @@ def tendencies(dates, time_increment, **kwargs):
             ################
 
             assert x.shape == c.shape, c.shape
-            print(
-                f"Computing data for {field.metadata('valid_datetime')}={field}-{b_field}"
-            )
+            print(f"Computing data for {field.metadata('valid_datetime')}={field}-{b_field}")
             out.write(x, template=field)
 
     out.close()
@@ -137,9 +135,7 @@ def tendencies(dates, time_increment, **kwargs):
     """
     )["config"]
 
-    dates = yaml.safe_load(
-        "[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]"
-    )
+    dates = yaml.safe_load("[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]")
     dates = to_datetime_list(dates)
 
     DEBUG = True

ecml_tools/create/functions/steps/rotate_winds.py

@@ -25,9 +25,7 @@ def rotate_winds(lats, lons, x_wind, y_wind, source_projection, target_projectio
     source_projection = pyproj.Proj(source_projection)
     target_projection = pyproj.Proj(target_projection)
 
-    transformer = pyproj.transformer.Transformer.from_proj(
-        source_projection, target_projection
-    )
+    transformer = pyproj.transformer.Transformer.from_proj(source_projection, target_projection)
 
     # To compute the new vector components:
     # 1) perturb each position in the direction of the winds
@@ -127,11 +125,7 @@ def execute(
             lons,
             x.to_numpy(reshape=False),
             y.to_numpy(reshape=False),
-            (
-                source_projection
-                if source_projection is not None
-                else CRS.from_cf(x.grid_mapping)
-            ),
+            (source_projection if source_projection is not None else CRS.from_cf(x.grid_mapping)),
             target_projection,
         )