Merge pull request #4 from ecmwf/chore/multiple-fixes-ci-precommit

Chore/multiple fixes ci precommit

.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

.github/workflows/changelog-pr-update.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - develop
+    paths-ignore:
+      - .pre-commit-config.yaml
+      - .readthedocs.yaml
 jobs:
   Check-Changelog:
     name: Check Changelog Action

.github/workflows/changelog-release-update.yml

@@ -0,0 +1,36 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+
+  workflow_dispatch: ~
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.release.target_commitish }}
+
+      - name: Update Changelog
+        uses: stefanzweifel/changelog-updater-action@v1
+        with:
+          latest-version: ${{ github.event.release.tag_name }}
+          heading-text: ${{ github.event.release.name }}
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          branch: docs/changelog-update-${{ github.event.release.tag_name }}
+          title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+          add-paths: |
+            CHANGELOG.md

.github/workflows/python-publish.yml

@@ -1,56 +1,26 @@
+---
 # This workflow will upload a Python Package using Twine when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
 name: Upload Python Package
 
 on:
-
-  push: {}
-  pull_request:
   release:
     types: [created]
 
 jobs:
   quality:
-    name: Code QA
-    runs-on: ubuntu-latest
-    steps:
-    - run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: 3.x
-    - uses: pre-commit/action@v3.0.1
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
 
   checks:
     strategy:
-      fail-fast: false
       matrix:
-        platform: ["ubuntu-latest", "macos-latest"]
-        python-version: ["3.10"]
-
-    name: Python  ${{ matrix.python-version }} on ${{ matrix.platform }}
-    runs-on: ${{ matrix.platform }}
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install
-      run: |
-        pip install -e .[all,tests]
-        pip freeze
-
-    - name: Tests
-      run: |
-        # disable tests on github to avoid giving away the token
-        # cd tests && python3 test_all.py
+        python-version: ["3.9", "3.10"]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
 
   deploy:
-
     needs: [checks, quality]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
     secrets: inherit

.github/workflows/python-pull-request.yml

@@ -0,0 +1,19 @@
+---
+# This workflow will upload a Python Package using Twine when a release is created from main
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2

.gitignore

@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/
@@ -186,3 +187,7 @@ _build/
 *.sync
 _version.py
 *.code-workspace
+
+
+# Environments
+.envrc

.pre-commit-config.yaml

@@ -20,6 +20,12 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
   rev: 24.10.0
   hooks:
@@ -62,3 +68,17 @@ repos:
   rev: "v2.5.0"
   hooks:
   - id: pyproject-fmt
+-   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
+    rev: v0.60.1
+    hooks:
+    -   id: docsig
+        args:
+        - --ignore-no-params # Allow docstrings without parameters
+        - --check-dunders    # Check dunder methods
+        - --check-overridden # Check overridden methods
+        - --check-protected  # Check protected methods
+        - --check-class      # Check class docstrings
+        - --disable=E113     # Disable empty docstrings
+        - --summary          # Print a summary
+ci:
+  autoupdate_schedule: monthly

pyproject.toml

@@ -8,20 +8,13 @@
 # nor does it submit to any jurisdiction.
 
 [build-system]
-requires = [
-  "setuptools>=60",
-  "setuptools-scm>=8",
-]
+requires = [ "setuptools>=60", "setuptools-scm>=8" ]
 
 [project]
 name = "anemoi-registry"
 
 description = "A package to manage a registry or data-driven forecasts."
-keywords = [
-  "ai",
-  "registry",
-  "tools",
-]
+keywords = [ "ai", "registry", "tools" ]
 
 license = { file = "LICENSE" }
 authors = [
@@ -45,15 +38,9 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 
-dynamic = [
-  "version",
-]
+dynamic = [ "version" ]
 
-dependencies = [
-  "anemoi-datasets",
-  "jsonpatch",
-  "requests",
-]
+dependencies = [ "anemoi-datasets", "jsonpatch", "requests" ]
 
 optional-dependencies.all = [
   "boto3",
@@ -92,9 +79,7 @@ urls.Repository = "https://github.com/ecmwf/anemoi-registry/"
 scripts.anemoi-registry = "anemoi.registry.__main__:main_PYTHON_ARGCOMPLETE_OK"
 
 [tool.setuptools.package-data]
-"anemoi.registry" = [
-  "*.yaml",
-]
+"anemoi.registry" = [ "*.yaml" ]
 
 [tool.setuptools_scm]
 version_file = "src/anemoi/registry/_version.py"

src/anemoi/registry/entry/__init__.py

@@ -28,6 +28,8 @@ class CatalogueEntryNotFound(Exception):
 
 
 class CatalogueEntry:
+    """Base class for a Anemoi catalogue entry."""
+
     record = None
     path = None
     key = None

src/anemoi/registry/entry/dataset.py

@@ -28,6 +28,8 @@
 
 
 class DatasetCatalogueEntryList(RestItemList):
+    """List of dataset catalogue entries."""
+
     def __init__(self, **kwargs):
         super().__init__(COLLECTION, **kwargs)
 
@@ -37,6 +39,8 @@ def __iter__(self):
 
 
 class DatasetCatalogueEntry(CatalogueEntry):
+    """A dataset catalogue entry."""
+
     collection = COLLECTION
     main_key = "name"
 

src/anemoi/registry/entry/experiment.py

@@ -31,6 +31,8 @@
 
 
 class ExperimentCatalogueEntryList(RestItemList):
+    """List of ExperimentCatalogueEntry objects."""
+
     def __init__(self, **kwargs):
         super().__init__(COLLECTION, **kwargs)
 
@@ -40,6 +42,8 @@ def __iter__(self):
 
 
 class ExperimentCatalogueEntry(CatalogueEntry):
+    """Catalogue entry for an experiment."""
+
     collection = COLLECTION
     main_key = "expver"
 

src/anemoi/registry/entry/weights.py

@@ -25,6 +25,8 @@
 
 
 class WeightsCatalogueEntryList(RestItemList):
+    """List of weights catalogue entries."""
+
     def __init__(self, **kwargs):
         super().__init__(COLLECTION, **kwargs)
 

src/anemoi/registry/rest.py

@@ -64,6 +64,8 @@ def trace_info():
 
 
 class Rest:
+    """REST API client."""
+
     def __init__(self):
         self.session = requests.Session()
         self.session.headers.update({"Authorization": f"Bearer {self.token}"})
@@ -154,6 +156,8 @@ def raise_for_status(self, r, errors={}):
 
 
 class RestItem:
+    """Single catalogue entry from REST API."""
+
     def __init__(self, collection, key):
         self.collection = collection
         self.key = key
@@ -188,6 +192,8 @@ def __repr__(self):
 
 
 class RestItemList:
+    """List of catalogue entries from REST API."""
+
     def __init__(self, collection):
         self.collection = collection
         self.rest = Rest()

src/anemoi/registry/tasks.py

@@ -23,6 +23,8 @@
 
 
 class TaskCatalogueEntryList:
+    """List of task catalogue entries."""
+
     collection = "tasks"
     main_key = "uuid"
 

src/anemoi/registry/workers/__init__.py

@@ -25,6 +25,8 @@
 
 
 class Worker:
+    """Base class for a worker that processes tasks in the queue."""
+
     name = None
 
     def __init__(

src/anemoi/registry/workers/delete_dataset.py

@@ -19,6 +19,8 @@
 
 
 class DeleteDatasetWorker(Worker):
+    """Worker to delete a dataset from a platform."""
+
     name = "delete-dataset"
 
     def __init__(

src/anemoi/registry/workers/dummy.py

@@ -16,6 +16,8 @@
 
 
 class DummyWorker(Worker):
+    """A debug worker that logs the task it receives."""
+
     name = "dummy"
 
     def __init__(self, arg, **kwargs):

src/anemoi/registry/workers/transfer_dataset.py

@@ -21,6 +21,8 @@
 
 
 class Progress:
+    """Progress reporter for transfer tasks."""
+
     latest = None
 
     def __init__(self, task, frequency=60):
@@ -65,6 +67,8 @@ def __call__(self, number_of_files, total_size, total_transferred, transfering,
 
 
 class TransferDatasetWorker(Worker):
+    """Worker to transfer a dataset from one platform to another."""
+
     name = "transfer-dataset"
 
     def __init__(