Merge branch 'develop' of github.com:MeteoSwiss-APN/anemoi-datasets i…

…nto develop
MeteoSwiss · Dec 2, 2024 · a4586d5 · a4586d5
2 parents 337b43a + 26a32a1
commit a4586d5
Show file tree

Hide file tree

Showing 147 changed files with 3,067 additions and 619 deletions.
diff --git a/.github/workflows/changelog-release-update.yml b/.github/workflows/changelog-release-update.yml
@@ -2,9 +2,11 @@
 name: "Update Changelog"
 
 on:
-  release:
-    types: [released]
-  workflow_dispatch: ~
+  workflow_run:
+    workflows:
+      - Upload Python Package
+    types:
+      - completed
 
 permissions:
   pull-requests: write
@@ -13,6 +15,7 @@ permissions:
 jobs:
   update:
     runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
 
     steps:
     - name: Checkout code
@@ -25,11 +28,18 @@ jobs:
       with:
         latest-version: ${{ github.event.release.tag_name }}
         heading-text: ${{ github.event.release.name }}
+        release-notes: ${{ github.event.release.body }}
 
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v6
       with:
         branch: docs/changelog-update-${{ github.event.release.tag_name }}
+        base: develop
         title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+        body: |
+          This PR updates the changelog to include the changes in the latest release.
+
+          > [!CAUTION]
+          > Merge DO NOT squash to correctly update the tag version of `develop` branch.
         add-paths: |
           CHANGELOG.md
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -42,13 +42,20 @@ jobs:
     with:
       anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
+      skip_matrix_jobs: |
+        gnu@debian-11
+        clang@rocky-8.6
+        gnu@debian-11
+        gnu@rocky-8.6
+        clang@rocky-8.6
+        gnu@fedora-37
     secrets: inherit
 
-   # Build downstream packages on HPC
-  downstream-ci-hpc:
-    name: downstream-ci-hpc
-    if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
-    with:
-      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
-    secrets: inherit
+#   # Build downstream packages on HPC
+#  downstream-ci-hpc:
+#    name: downstream-ci-hpc
+#    if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+#    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+#    with:
+#      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
+#    secrets: inherit
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -15,9 +15,12 @@ jobs:
       skip-hooks: "no-commit-to-branch"
 
   checks:
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
-      python-version: "3.9"
+      python-version: ${{ matrix.python-version }}
 
   deploy:
     needs: [checks, quality]

diff --git a/.github/workflows/python-pull-request.yml b/.github/workflows/python-pull-request.yml
@@ -1,12 +1,17 @@
 # This workflow will upload a Python Package using Twine when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
-name: Code Quality checks for PRs
+name: Test PR
 
 on:
-  push:
   pull_request:
     types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - develop
+  schedule:
+    - cron: "9 2 * * 0" # at 9:02 on sunday
+
 
 jobs:
   quality:
@@ -17,7 +22,7 @@ jobs:
   checks:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.11"]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -27,7 +27,7 @@ repos:
     -   id: python-check-blanket-noqa # Check for # noqa: all
     -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.8.0
+  rev: 24.10.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -40,16 +40,15 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.9
+  rev: v0.7.2
   hooks:
   - id: ruff
-    # Next line if for documenation cod snippets
-    exclude: '.*/[^_].*_\.py$'
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
     - --preview
+    - --exclude=docs/**/*_.py
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v1.0.0
   hooks:
@@ -60,13 +59,8 @@ repos:
   hooks:
   - id: rstfmt
     exclude: 'cli/.*' # Because we use argparse
-- repo: https://github.com/b8raoult/pre-commit-docconvert
-  rev: "0.1.5"
-  hooks:
-  - id: docconvert
-    args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.2.4"
+  rev: "v2.5.0"
   hooks:
   - id: pyproject-fmt
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,19 +8,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
-## [Unreleased](https://github.com/ecmwf/anemoi-datasets/compare/0.5.7...HEAD)
+## [Unreleased](https://github.com/ecmwf/anemoi-datasets/compare/0.5.8...HEAD)
 
+## Changed
+
+- Fix metadata serialization handling of numpy.integer (#140)
+- Fix cutout slicing of grid dimension (#145)
+
+### Added
+
+- Call filters from anemoi-transform
+- make test optional when adls is not installed Pull request #110
+
+## [0.5.8](https://github.com/ecmwf/anemoi-datasets/compare/0.5.7...0.5.8) - 2024-10-26
+
+### Changed
+
+- Bugfix in `auto_adjust`
+- Fixed precommit CI errors
+- Improve tests
+- More verbosity
 
 ### Added
 
 - Add anemoi-transform link to documentation
 - Various bug fixes
 - Control compatibility check in xy/zip
 - Add `merge` feature
+- Add support for storing `supporting_arrays` in checkpoint files
+- Allow naming of datasets components
+- Contributors file (#105)
 
 ### Changed
 
+- Upload with ssh (experimental)
 - Remove upstream dependencies from downstream-ci workflow (temporary) (#83)
+- ci: pin python versions to 3.9 ... 3.12 for checks (#93)
+- Fix `__version__` import in init
 
 ## [0.5.7](https://github.com/ecmwf/anemoi-datasets/compare/0.5.6...0.5.7) - 2024-10-09
 

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -0,0 +1,13 @@
+## How to Contribute
+
+Please see the [read the docs](https://anemoi-training.readthedocs.io/en/latest/dev/contributing.html).
+
+
+## Contributors
+
+Thank you to all the wonderful people who have contributed to Anemoi. Contributions can come in many forms, including code, documentation, bug reports, feature suggestions, design, and more. A list of code-based contributors can be found [here](https://github.com/ecmwf/anemoi-datasets/graphs/contributors).
+
+
+## Contributing Organisations
+
+Significant contributions have been made by the following organisations: [DWD](https://www.dwd.de/), [MET Norway](https://www.met.no/), [MeteoSwiss](https://www.meteoswiss.admin.ch/), [RMI](https://www.meteo.be/) & [ECMWF](https://www.ecmwf.int/)
diff --git a/README.md b/README.md
@@ -22,7 +22,7 @@ $ pip install anemoi-datasets
 ## License
 
 ```
-Copyright 2022, European Centre for Medium Range Weather Forecasts.
+Copyright 2024, Anemoi contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/docs/building/sources/xarray-zarr.rst b/docs/building/sources/xarray-zarr.rst
@@ -2,5 +2,17 @@
  xarray-zarr
 #############
 
+Here is an example recipe that builds a dataset using one of the many
+regridded versions of ERA5 hosted by Google in Analysis-Ready, Cloud
+Optimized format. See `here
+<https://cloud.google.com/storage/docs/public-datasets/era5>`_ for more
+information.
+
 .. literalinclude:: yaml/xarray-zarr.yaml
    :language: yaml
+
+Note that unlike the ``mars`` examples, there is no need to include a
+``grid`` specification. Also, in order to subselect the vertical levels,
+it is necessary to use the :ref:`join <building-join>` operation to join
+separate lists containing 2D variables and 3D variables. If all vertical
+levels are desired, then it is OK to specify a single source.
diff --git a/docs/building/sources/yaml/xarray-zarr.yaml b/docs/building/sources/yaml/xarray-zarr.yaml
@@ -1,3 +1,22 @@
+dates:
+  start: 2021-01-01T00:00:00
+  end: 2021-01-10T12:00:00
+  frequency: 6h
+
 input:
-  xarray-zarr:
-    url: https://...
+  join:
+      - xarray-zarr:
+          url: "gs://gcp-public-data-arco-era5/ar/1959-2022-6h-128x64_equiangular_conservative.zarr"
+          param:
+              - surface_pressure
+              - 2m_temperature
+              - 10m_u_component_of_wind
+              - 10m_v_component_of_wind
+
+      - xarray-zarr:
+          url: "gs://gcp-public-data-arco-era5/ar/1959-2022-6h-128x64_equiangular_conservative.zarr"
+          param:
+              - temperature
+          level:
+              - 1000
+              - 500
diff --git a/docs/conf.py b/docs/conf.py
@@ -34,15 +34,15 @@
 
 project = "Anemoi"
 
-author = "ECMWF"
+author = "Anemoi contributors"
 
 year = datetime.datetime.now().year
 if year == 2024:
     years = "2024"
 else:
     years = "2024-%s" % (year,)
 
-copyright = "%s, ECMWF" % (years,)
+copyright = "%s, Anemoi contributors" % (years,)
 
 
 try:

diff --git a/docs/using/missing.rst b/docs/using/missing.rst
@@ -4,6 +4,16 @@
  Managing missing dates
 ########################
 
+*********************************************
+ Managing missing dates with anemoi-training
+*********************************************
+
+Anemoi-training has internal handling of missing dates, and will
+calculate the valid date indices used during training using the
+``missing`` property. Consequenctly, when training a model with
+anemoi-training, you should `not` specify a method to deal with missing
+dates in the dataloader configuration file.
+
 **************************************************
  Filling the missing dates with artificial values
 **************************************************

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,14 +1,12 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
-
 [build-system]
 requires = [
   "setuptools>=60",
@@ -42,6 +40,7 @@ classifiers = [
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -50,7 +49,8 @@ dynamic = [
   "version",
 ]
 dependencies = [
-  "anemoi-utils[provenance]>=0.3.18",
+  "anemoi-transform>=0.1",
+  "anemoi-utils[provenance]>=0.4.9",
   "cfunits",
   "numpy",
   "pyyaml",
@@ -96,6 +96,9 @@ optional-dependencies.tests = [
 optional-dependencies.xarray = [
   "gcsfs",
   "kerchunk",
+  "pandas",
+  "planetary-computer",
+  "pystac-client",
 ]
 
 urls.Documentation = "https://anemoi-datasets.readthedocs.io/"

diff --git a/src/anemoi/datasets/__init__.py b/src/anemoi/datasets/__init__.py
@@ -1,19 +1,27 @@
-# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-from ._version import __version__
 from .data import MissingDateError
 from .data import add_dataset_path
 from .data import add_named_dataset
 from .data import list_dataset_names
 from .data import open_dataset
 
+try:
+    # NOTE: the `_version.py` file must not be present in the git repository
+    #   as it is generated by setuptools at install time
+    from ._version import __version__  # type: ignore
+except ImportError:  # pragma: no cover
+    # Local copy or not installed with setuptools
+    __version__ = "999"
+
 __all__ = [
-    "__version__",
     "add_dataset_path",
     "add_named_dataset",
     "list_dataset_names",

diff --git a/src/anemoi/datasets/__main__.py b/src/anemoi/datasets/__main__.py
@@ -1,12 +1,11 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
-#
 
 from anemoi.utils.cli import cli_main
 from anemoi.utils.cli import make_parser