Merge branch 'main' into 145/fcis-architecture

ACCESS-NRI · Nov 7, 2024 · 4b471a7 · 4b471a7
2 parents 2a872f0 + 495aaa5
commit 4b471a7
Show file tree

Hide file tree

Showing 13 changed files with 437 additions and 273 deletions.
diff --git a/conda/environment.yml → .conda/env_build.yml b/conda/environment.yml → .conda/env_build.yml
@@ -1,16 +1,11 @@
-name: um2nc
-
 channels:
-    - conda-forge
     - accessnri
+    - conda-forge
     - coecms
-    - defaults
+    - nodefaults
 
 dependencies:
     - anaconda-client
     - conda-build
     - conda-verify
-    - versioneer
-    - pylint
-    - pytest
-    - pytest-cov
+    - versioneer
diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml
@@ -0,0 +1,25 @@
+name: um2nc-dev
+
+channels:
+    - accessnri
+    - conda-forge
+    - coecms
+    - nodefaults
+
+
+dependencies:
+    - python >=3.10,<=3.12
+    - cf-units
+    - cftime
+    - f90nml
+    - ipykernel
+    - mule
+    - netcdf4
+    - numpy <2
+    - pip
+    - pytest
+    - pytest-cov
+    - pylint #TODO: remove this after setting up linting with ruff
+    - ruff
+    - scitools-iris
+    - versioneer
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -0,0 +1,45 @@
+{% set version = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True).get('version') %}
+{% set project = load_file_data('../pyproject.toml', from_recipe_dir=True).get('project') %}
+
+package:
+    name: {{ project.get('name') }}
+    version: "{{ version }}"
+
+build:
+    noarch: python
+    number: 0
+    script: "python3 -m pip install . -vv"
+    entry_points:
+        {% for name, script in project.get('scripts').items() %}
+        - {{ name }} = {{ script }}
+        {% endfor %}
+
+source:
+    path: ../
+
+requirements:
+    host:
+        - python
+        - pip
+        - setuptools >=61.0.0
+        - versioneer
+    run:
+        {% for dep in project.get('dependencies') %}
+        - {{ dep }}
+        {% endfor %}
+
+test:
+    imports:
+        - umpost
+    commands:
+    {% for name, script in project.get('scripts').items() %}
+        - {{ name }} --help
+    {% endfor %}
+
+about:
+    home: {{ project.get('urls').get('Repository') }}
+    license: Apache Software
+    license_file: LICENSE
+    summary: {{ project.get('description') }}
+    license_family: Apache
+
diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml
@@ -6,7 +6,7 @@ on:
         - '*'
 
 env:
-  PY_VERSION: "3.10"
+  PY_VERSION: "3.11"
 
 jobs:
 
@@ -25,7 +25,7 @@ jobs:
         with:
           miniconda-version: "latest"
           python-version: ${{ env.PY_VERSION }}
-          environment-file: conda/environment.yml
+          environment-file: conda/env_build.yml
           auto-update-conda: false
           auto-activate-base: false
           show-channel-urls: true

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -5,85 +5,95 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   push:
-    branches: [ main, develop ]
+    branches: main
   pull_request:
-    branches: [ main, develop ]
-
+    branches: main
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
-
 env:
-  PY_VERSION: "3.10"
+  PY_VERSION: "3.11"
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  conda-build:
-    name: Conda Build
+  # JOB to run change in the build files
+  changes:
     runs-on: ubuntu-latest
-    env:
-      NAME: test-${{ github.event.repository.name }}
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
     outputs:
-      artifact-name: ${{ env.NAME }}
+      files: ${{ steps.filter.outputs.files }}
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+
+      - name: Filter files
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2
+        id: filter
+        with:
+          filters: |
+            files:
+              - 'setup.py'
+              - 'pyproject.toml'
+              - '.conda/env_build.yml'
+              - '.conda/meta.yml'
+
+  verify-conda-build:
+    name: Conda Build
+    runs-on: ubuntu-latest
+    needs: changes
+    # Only run if there are changes in the build files
+    if: ${{ needs.changes.outputs.files == 'true' }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup conda environment
-        uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1
+      - name: Setup conda build environment
+        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
         with:
           miniconda-version: "latest"
-          python-version: ${{ env.PY_VERSION }}
-          environment-file: conda/environment.yml
-          auto-update-conda: false
+          python-version: ${{ env.PY_VERSION }} 
+          environment-file: .conda/env_build.yml
           auto-activate-base: false
+          auto-update-conda: false
           show-channel-urls: true
 
-      - name: Run conda build
+      - name: Verify conda recipe
         shell: bash -el {0}
-        run: |
-          conda build . --no-anaconda-upload --output-folder=./build
+        run: conda-verify .conda --ignore C2105,C2122
 
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ${{ env.NAME }}
-          if-no-files-found: error
-          path: ./build
+      - name: Run conda build
+        shell: bash -el {0}
+        run: conda build . --no-anaconda-upload --output-folder=./build -c conda-forge -c accessnri -c coecms
 
+      - name: Verify conda package
+        shell: bash -el {0}
+        run: conda-verify ./build/noarch/*.tar.bz2 --ignore C1105,C1115,C1141
+
   tests:
     name: Tests
     runs-on: ubuntu-latest
-    needs: conda-build
-
-    # Run the job for different versions of python
     strategy:
+      fail-fast: true
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
-
-    env:
-      ARTIFACT_LOCATION: ${{ github.workspace }}/conda-local
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-
       - name: Checkout code
         uses: actions/[email protected]
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: ${{ needs.conda-build.outputs.artifact-name }}
-          path: ${{ env.ARTIFACT_LOCATION }}
-
       - name: Setup conda environment
-        uses: conda-incubator/setup-miniconda@v3
+        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
-          environment-file: conda/environment.yml
-          activate-environment: um2nc
+          environment-file: .conda/env_dev.yml
+          activate-environment: um2nc-dev
+          auto-update-conda: false
+          auto-activate-base: false
+          show-channel-urls: true
 
-      - name: Install conda package
+      - name: Install source
         shell: bash -l {0}
-        run: |
-          conda install -c file://${{ env.ARTIFACT_LOCATION }} -c conda-forge -c accessnri -c coecms um2nc
+        run: python3 -m pip install --no-deps --no-build-isolation -e .
 
       - name: List installed packages
         shell: bash -l {0}
@@ -103,12 +113,12 @@ jobs:
 
       - name: Run tests
         shell: bash -l {0}
-        run: python -m pytest --cov=umpost --cov-report=xml -s test
-
-      - name: Upload code coverage
-        uses: codecov/codecov-action@v4
-        # Only upload once for the installed version
-        if: matrix.python-version == ${{ env.PY_VERSION }}
-        with:
-          token: ${{ secrets.codecov_token }}
-          files: ./coverage.xml
+        run: python -m pytest --cov=umpost --cov-report=html -s test
+
+      # - name: Upload code coverage
+      #   # Only upload once for the installed version
+      #   if: ${{ matrix.python-version == env.PY_VERSION }}
+      #   uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 #v4.6.0
+      #   with:
+      #     token: ${{ secrets.codecov_token }}
+      #     files: ./coverage.xml
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,6 @@
 .idea
 .coverage
 *~
+*.egg-info
+coverage_html
+*.code-workspace
diff --git a/README.md b/README.md
@@ -1,37 +1,72 @@
-# Unified Model Post-processing: um2nc-standalone
+# Unified Model to NetCDF Post-processing: um2nc
 
 ## About
 
-`um2nc-standalone` is an [ACCESS-NRI](https://www.access-nri.org.au/) project to merge multiple versions of unified model conversion tools to a single, canonical project for the ESM1.5 model. 
+`um2nc` is a `Python3` utility to convert [Unified Model data files](https://code.metoffice.gov.uk/doc/um/latest/papers/umdp_F03.pdf) to NetCDF format.
+
+The `um2nc-standalone` project is an [ACCESS-NRI](https://www.access-nri.org.au/) initiative to merge multiple versions of Unified Model NetCDF conversion tool to a single, canonical project. 
 
 ## Installation
 
-TODO
+### Gadi
 
-* `virtualenv` instructions
-* `conda`/`miniconda`/`micromamba?` instructions
+On Gadi, `um2nc` is available within the `vk83` `payu` environment. 
+To access it, run: 
+```
+module use /g/data/vk83/modules
+module load payu
+```
+> [!IMPORTANT]  
+> You need to be a member of the vk83 project on NCI to access the module. For more information check how to [Join relevant NCI projects](https://access-hive.org.au/getting_started/set_up_nci_account/#join-relevant-nci-projects)
 
-## User documentation
+### Local installation
+`um2nc` is available as a `conda` package in the [access-nri conda channel](https://anaconda.org/accessnri/um2nc).
+To install it run:
+```
+conda install accessnri::um2nc
+```
 
-TODO
+## Development/Testing instructions
+For development/testing, it is recommended to install `um2nc` as a development package within a `micromamba`/`conda` testing environment.
 
-### Running the tests
+### Clone um2nc-standalone GitHub repo
+```
+git clone [email protected]:ACCESS-NRI/um2nc-standalone.git
+```
 
-This project uses `pytest`. To run the tests:
+### Create a micromamba/conda testing environment
+> [!TIP]  
+> In the following instructions `micromamba` can be replaced with `conda`.
 
-```Bash
-$ cd <your um2nc-standalone dir>
-$ pytest
+```
+cd um2nc-standalone
+micromamba env create -n um2nc_dev --file .conda/env_dev.yml
+micromamba activate um2nc_dev
+```
+
+### Install um2nc as a development package
+```
+pip install --no-deps --no-build-isolation -e .
 ```
 
-A minimal code coverage setup has been included, to run & generate an HTML coverage report:
+### Running the tests
+
+The `um2nc-standalone` project uses `pytest` and `pytest-cov`.<br>
+To run the tests and generate print a coverage report (with missing lines) run:
 
 ```
-$ cd <your um2nc-standalone dir>
-$ pytest --cov-report=html --cov=umpost
+python3 -m pytest --cov-report=term-missing --cov=umpost
 ```
+> [!TIP]
+> To generate an HTML coverage report substitute `term-missing` with `html`.
+
+## User documentation
+
+TODO: this needs to cover:
 
-Then load the `index.html` from the project root/coverage_html dir.
+1. Running `um2netcdf` standalone
+2. Using the workflow run script
+3. Using `um2netcdf` as an API
 
 ## Further information