Chore/multiple fixes ci pre commit

.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# CODEOWNERS file
+
+# Protect workflow files
+/.github/ @theissenhelen @jesperdramsch @gmertes
+/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes
+/pyproject.toml @theissenhelen @jesperdramsch @gmertes

.github/ci-hpc-config.yml

@@ -0,0 +1,8 @@
+build:
+  python: '3.10'
+  modules:
+    - ninja
+  parallel: 64
+
+  pytest_cmd: |
+    python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml

.github/workflows/changelog-pr-update.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - develop
+    paths-ignore:
+      - .pre-commit-config.yaml
+      - .readthedocs.yaml
 jobs:
   Check-Changelog:
     name: Check Changelog Action

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
   downstream-ci-hpc:
     name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit

.gitignore

@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/

.pre-commit-config.yaml

@@ -20,8 +20,14 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.4.2
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -34,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     # Next line if for documenation cod snippets
@@ -45,7 +51,7 @@ repos:
     - --exit-non-zero-on-fix
     - --preview
 - repo: https://github.com/sphinx-contrib/sphinx-lint
-  rev: v0.9.1
+  rev: v1.0.0
   hooks:
   - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
@@ -59,12 +65,23 @@ repos:
   hooks:
   - id: docconvert
     args: ["numpy"]
-- repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.6"
-  hooks:
-  - id: optional-dependencies-all
-    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.1.3"
+  rev: "2.2.3"
   hooks:
   - id: pyproject-fmt
+-   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
+    rev: v0.60.1
+    hooks:
+    -   id: docsig
+        args:
+        - --ignore-no-params # Allow docstrings without parameters
+        - --check-dunders    # Check dunder methods
+        - --check-overridden # Check overridden methods
+        - --check-protected  # Check protected methods
+        - --check-class      # Check class docstrings
+        - --disable=E113     # Disable empty docstrings
+        - --summary          # Print a summary
+ci:
+  autoupdate_schedule: monthly
+ci:
+  autoupdate_schedule: monthly

CHANGELOG.md

@@ -11,9 +11,14 @@ Keep it human-readable, your future self will thank you!
 ## [Unreleased]
 
 ### Added
+- Codeowners file
+- Pygrep precommit hooks
+- Docsig precommit hooks
+- Changelog merge strategy- Codeowners file
 
 ### Changed
 - downstream-ci should only runs for changes in src and tests
+- bugfixes for CI
 
 ### Removed
 

pyproject.toml

@@ -10,19 +10,13 @@
 # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
 
 [build-system]
-requires = [
-  "setuptools>=60",
-  "setuptools-scm>=8",
-]
+requires = [ "setuptools>=60", "setuptools-scm>=8" ]
 
 [project]
 name = "anemoi-utils"
 
 description = "A package to hold various functions to support training of ML models on ECMWF data."
-keywords = [
-  "ai",
-  "tools",
-]
+keywords = [ "ai", "tools" ]
 
 license = { file = "LICENSE" }
 authors = [
@@ -45,34 +39,16 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 
-dynamic = [
-  "version",
-]
+dynamic = [ "version" ]
 dependencies = [
   "aniso8601",
   "pyyaml",
   "tomli",     # Only needed before 3.11
   "tqdm",
 ]
 
-optional-dependencies.all = [
-  "gitpython",
-  "nvsmi",
-  "requests",
-  "termcolor",
-]
-optional-dependencies.dev = [
-  "gitpython",
-  "nbsphinx",
-  "nvsmi",
-  "pandoc",
-  "pytest",
-  "requests",
-  "sphinx",
-  "sphinx-argparse<0.5",
-  "sphinx-rtd-theme",
-  "termcolor",
-]
+optional-dependencies.all = [ "anemoi-utils[grib,provenance,text]" ]
+optional-dependencies.dev = [ "anemoi-utils[all,docs,tests]" ]
 
 optional-dependencies.docs = [
   "nbsphinx",
@@ -84,21 +60,14 @@ optional-dependencies.docs = [
   "termcolor",
 ]
 
-optional-dependencies.grib = [
-  "requests",
-]
+optional-dependencies.grib = [ "requests" ]
 
-optional-dependencies.provenance = [
-  "gitpython",
-  "nvsmi",
-]
-optional-dependencies.tests = [
-  "pytest",
-]
+optional-dependencies.provenance = [ "gitpython", "nvsmi" ]
+
+optional-dependencies.tests = [ "pytest" ]
+
+optional-dependencies.text = [ "termcolor" ]
 
-optional-dependencies.text = [
-  "termcolor",
-]
 urls.Documentation = "https://anemoi-utils.readthedocs.io/"
 urls.Homepage = "https://github.com/ecmwf/anemoi-utils/"
 urls.Issues = "https://github.com/ecmwf/anemoi-utils/issues"
@@ -107,9 +76,7 @@ urls.Repository = "https://github.com/ecmwf/anemoi-utils/"
 scripts.anemoi-utils = "anemoi.utils.__main__:main"
 
 [tool.setuptools.package-data]
-"anemoi.utils.mars" = [
-  "*.yaml",
-]
+"anemoi.utils.mars" = [ "*.yaml" ]
 
 [tool.setuptools_scm]
 version_file = "src/anemoi/utils/_version.py"

src/anemoi/utils/caching.py

@@ -49,6 +49,7 @@ def cache(key, proc, collection="default", expires=None):
 
 
 class cached:
+    """Decorator to cache the result of a function."""
 
     def __init__(self, collection="default", expires=None):
         self.collection = collection

src/anemoi/utils/checkpoints.py

@@ -47,7 +47,7 @@ def has_metadata(path: str, name: str = DEFAULT_NAME) -> bool:
     return False
 
 
-def load_metadata(path: str, name: str = DEFAULT_NAME):
+def load_metadata(path: str, name: str = DEFAULT_NAME) -> dict:
     """Load metadata from a checkpoint file
 
     Parameters
@@ -59,8 +59,8 @@ def load_metadata(path: str, name: str = DEFAULT_NAME):
 
     Returns
     -------
-    JSON
-        The content of the metadata file
+    dict
+        The content of the metadata file from JSON
 
     Raises
     ------
@@ -82,7 +82,7 @@ def load_metadata(path: str, name: str = DEFAULT_NAME):
         raise ValueError(f"Could not find '{name}' in {path}.")
 
 
-def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER):
+def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER) -> None:
     """Save metadata to a checkpoint file
 
     Parameters
@@ -93,6 +93,8 @@ def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER):
         A JSON serializable object
     name : str, optional
         The name of the metadata file in the zip archive
+    folder : str, optional
+        The folder where the metadata file will be saved
     """
     with zipfile.ZipFile(path, "a") as zipf:
 

src/anemoi/utils/cli.py

@@ -55,6 +55,8 @@ def make_parser(description, commands):
 
 
 class Failed(Command):
+    """Command not available."""
+
     def __init__(self, name, error):
         self.name = name
         self.error = error

src/anemoi/utils/config.py

@@ -175,9 +175,14 @@ def config_path(name="settings.toml"):
     return full
 
 
-def load_any_dict_format(path):
+def load_any_dict_format(path) -> dict:
     """Load a configuration file in any supported format: JSON, YAML and TOML.
 
+    Parameters
+    ----------
+    path : str
+        The path to the configuration file.
+
     Returns
     -------
     dict
@@ -243,7 +248,7 @@ def _load_config(name="settings.toml", secrets=None, defaults=None):
     return CONFIG[key]
 
 
-def _save_config(name, data):
+def _save_config(name, data) -> None:
     CONFIG.pop(name, None)
 
     conf = config_path(name)
@@ -265,7 +270,7 @@ def _save_config(name, data):
         f.write(data)
 
 
-def save_config(name, data):
+def save_config(name, data) -> None:
     """Save a configuration file.
 
     Parameters
@@ -281,13 +286,17 @@ def save_config(name, data):
         _save_config(name, data)
 
 
-def load_config(name="settings.toml", secrets=None, defaults=None):
+def load_config(name="settings.toml", secrets=None, defaults=None) -> DotDict | str:
     """Read a configuration file.
 
     Parameters
     ----------
     name : str, optional
         The name of the config file to read, by default "settings.toml"
+    secrets : str or list, optional
+        The name of the secrets file, by default None
+    defaults : str or dict, optional
+        The name of the defaults file, by default None
 
     Returns
     -------
@@ -299,7 +308,7 @@ def load_config(name="settings.toml", secrets=None, defaults=None):
         return _load_config(name, secrets, defaults)
 
 
-def load_raw_config(name, default=None):
+def load_raw_config(name, default=None) -> DotDict | str:
 
     path = config_path(name)
     if os.path.exists(path):
@@ -308,13 +317,17 @@ def load_raw_config(name, default=None):
     return default
 
 
-def check_config_mode(name="settings.toml", secrets_name=None, secrets=None):
+def check_config_mode(name="settings.toml", secrets_name=None, secrets=None) -> None:
     """Check that a configuration file is secure.
 
     Parameters
     ----------
     name : str, optional
         The name of the configuration file, by default "settings.toml"
+    secrets_name : str, optional
+        The name of the secrets file, by default None
+    secrets : list, optional
+        The list of secrets to check, by default None
 
     Raises
     ------