From 8b3900802ce5b8b46030f8e9749d6308ef3c189d Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 26 Sep 2024 16:58:17 +1000 Subject: [PATCH] Set up build and dev conda environment files. Modified pyproject.toml to reflect modifications in the conda environment files. Modified meta.yaml for conda packaging. Modified CI.yml for conda-build testing. Modified README.md with instructions on how to install (Gadi, locally and dev). --- conda/environment.yml => .conda/env_build.yml | 10 +- .conda/env_dev.yml | 25 +++++ .conda/meta.yaml | 45 +++++++++ .github/workflows/CI.yml | 91 +++++++++++-------- .gitignore | 2 + README.md | 91 ++++++++++++++++--- conda/meta.yaml | 41 --------- pyproject.toml | 38 ++++---- 8 files changed, 224 insertions(+), 119 deletions(-) rename conda/environment.yml => .conda/env_build.yml (59%) create mode 100644 .conda/env_dev.yml create mode 100644 .conda/meta.yaml delete mode 100644 conda/meta.yaml diff --git a/conda/environment.yml b/.conda/env_build.yml similarity index 59% rename from conda/environment.yml rename to .conda/env_build.yml index 87cf5b4..fce2122 100644 --- a/conda/environment.yml +++ b/.conda/env_build.yml @@ -1,16 +1,12 @@ -name: um2nc - channels: - conda-forge - accessnri - coecms - - defaults + - default + - nodefaults dependencies: - anaconda-client - conda-build - conda-verify - - versioneer - - pylint - - pytest - - pytest-cov + - versioneer \ No newline at end of file diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml new file mode 100644 index 0000000..135dc85 --- /dev/null +++ b/.conda/env_dev.yml @@ -0,0 +1,25 @@ +name: um2nc-dev + +channels: + - conda-forge + - accessnri + - coecms + - default + - nodefaults + +dependencies: + - python >=3.10,<=3.12 + - cf-units + - cftime + - f90nml + - ipykernel + - mule + - netcdf4 + - numpy + - pip + - pytest + - pytest-cov + - pylint #TODO: remove this after setting up linting with ruff + - ruff + - scitools-iris + - versioneer \ No newline at end of file diff --git a/.conda/meta.yaml b/.conda/meta.yaml new file mode 100644 index 0000000..f080ae5 --- /dev/null +++ b/.conda/meta.yaml @@ -0,0 +1,45 @@ +{% set version = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True).get('version') %} +{% set project = load_file_data('../pyproject.toml', from_recipe_dir=True).get('project') %} + +package: + name: {{ project.get('name') }} + version: "{{ version }}" + +build: + noarch: python + number: 0 + script: "python3 -m pip install . -vv" + entry_points: + {% for name, script in project.get('scripts').items() %} + - {{ name }} = {{ script }} + {% endfor %} + +source: + path: ../ + +requirements: + host: + - python + - pip + - setuptools >=61.0.0 + - versioneer + run: + {% for dep in project.get('dependencies') %} + - {{ dep }} + {% endfor %} + +test: + imports: + - umpost + commands: + {% for name, script in project.get('scripts', {}).items() %} + - {{ name }} --help + {% endfor %} + +about: + home: https://github.com/access-nri/um2nc-standalone + license: Apache Software + license_file: LICENSE + summary: "Tool to convert UM fieldsfiles into netCDF format." + license_family: Apache + diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 068d3e9..4181c6f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -8,82 +8,93 @@ on: branches: [ main, develop ] pull_request: branches: [ main, develop ] - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: - env: - PY_VERSION: "3.10" + PY_VERSION: 3.11 -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - conda-build: - name: Conda Build + # JOB to run change in the build files + changes: runs-on: ubuntu-latest - env: - NAME: test-${{ github.event.repository.name }} + # Required permissions + permissions: + pull-requests: read + # Set job outputs to values from filter step outputs: - artifact-name: ${{ env.NAME }} + files: ${{ steps.filter.outputs.files }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2 + id: filter + with: + filters: | + files: + - 'setup.py' + - 'pyproject.toml' + - '.conda/env_build.yml' + - '.conda/meta.yml' + + verify-conda-build: + name: Conda Build + runs-on: ubuntu-latest + needs: changes + # Only run if there are changes in the build files + if: ${{ needs.changes.outputs.files == 'true' }} steps: - uses: actions/checkout@v4 - - name: Setup conda environment - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 + - name: Setup conda build environment + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: miniconda-version: "latest" - python-version: ${{ env.PY_VERSION }} - environment-file: conda/environment.yml - auto-update-conda: false + python-version: ${{ env.PY_VERSION }} + environment-file: .conda/env_build.yml auto-activate-base: false + auto-update-conda: false show-channel-urls: true - - name: Run conda build + - name: Verify conda recipe shell: bash -el {0} - run: | - conda build . --no-anaconda-upload --output-folder=./build + run: conda-verify .conda --ignore C2105,C2122 - - uses: actions/upload-artifact@v4 - with: - name: ${{ env.NAME }} - if-no-files-found: error - path: ./build + - name: Run conda build + shell: bash -el {0} + run: conda build . --no-anaconda-upload --output-folder=./build -c conda-forge -c accessnri -c coecms + - name: Verify conda package + shell: bash -el {0} + run: conda-verify ./build/noarch/*.tar.bz2 --ignore C1105,C1115,C1141 + tests: name: Tests runs-on: ubuntu-latest - needs: conda-build - - # Run the job for different versions of python strategy: + fail-fast: true matrix: python-version: ["3.10", "3.11", "3.12"] - env: ARTIFACT_LOCATION: ${{ github.workspace }}/conda-local - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - - name: Checkout code uses: actions/checkout@v4.1.7 - - uses: actions/download-artifact@v4 - with: - name: ${{ needs.conda-build.outputs.artifact-name }} - path: ${{ env.ARTIFACT_LOCATION }} - - name: Setup conda environment - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: miniconda-version: "latest" python-version: ${{ matrix.python-version }} - environment-file: conda/environment.yml - activate-environment: um2nc + environment-file: .conda/env_dev.yml + activate-environment: um2nc-dev + auto-update-conda: false + auto-activate-base: false + show-channel-urls: true - - name: Install conda package + - name: Install source shell: bash -l {0} - run: | - conda install -c file://${{ env.ARTIFACT_LOCATION }} -c conda-forge -c accessnri -c coecms um2nc + run: python3 -m pip install --no-deps --no-build-isolation -e . - name: List installed packages shell: bash -l {0} diff --git a/.gitignore b/.gitignore index 644c4ad..e475ebd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ .idea .coverage *~ +*.egg-info +coverage_html \ No newline at end of file diff --git a/README.md b/README.md index 55559b6..7ff25f2 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,98 @@ -# Unified Model Post-processing: um2nc-standalone +# Unified Model to NetCDF Post-processing: um2nc ## About -`um2nc-standalone` is an [ACCESS-NRI](https://www.access-nri.org.au/) project to merge multiple versions of unified model conversion tools to a single, canonical project for the ESM1.5 model. +`um2nc` is a `Python3` utility to convert [Unified Model data files](https://code.metoffice.gov.uk/doc/um/latest/papers/umdp_F03.pdf) to NetCDF format. + +The `um2nc-standalone` project is an [ACCESS-NRI](https://www.access-nri.org.au/) initiative to merge multiple versions of Unified Model NetCDF conversion tool to a single, canonical project. ## Installation -TODO +### Gadi -* `virtualenv` instructions -* `conda`/`miniconda`/`micromamba?` instructions +On Gadi, `um2nc` is available within the `vk83` `payu` environment. +To access it, run: +``` +module use /g/data/vk83/modules +module load payu +``` +> [!IMPORTANT] +> You need to be a member of the vk83 project on NCI to access the module. For more information check how to [Join relevant NCI projects](https://access-hive.org.au/getting_started/set_up_nci_account/#join-relevant-nci-projects) -## User documentation +### Local installation +`um2nc` is available as a `conda` package in the [access-nri conda channel](https://anaconda.org/accessnri/um2nc). +To install it run: +``` +conda install accessnri::um2nc +``` -TODO +### Installing Dependencies +IS THIS REQUIRED?? +udunits is also available in conda-forge... -### Running the tests +#### MacOS + +```Bash +brew install udunits +``` + +if this fails with some compiler/header errors, try: + +```Bash +# set up non standard cf-unit paths +export UDUNITS2_XML_PATH=/opt/homebrew/Cellar/udunits/2.2.28/share/udunits/udunits2.xml +export UDUNITS2_LIBDIR=/opt/homebrew/lib +export UDUNITS2_INCDIR=/opt/homebrew/Cellar/udunits/2.2.28/include +brew install udunits +``` -This project uses `pytest`. To run the tests: +#### Linux ```Bash -$ cd -$ pytest +apt-get install udunits ``` -A minimal code coverage setup has been included, to run & generate an HTML coverage report: +## Development/Testing instructions +For development/testing, it is recommended to install `um2nc` as a development package within a `micromamba`/`conda` testing environment. +### Clone um2nc-standalone GitHub repo ``` -$ cd -$ pytest --cov-report=html --cov=umpost +git clone git@github.com:ACCESS-NRI/um2nc-standalone.git ``` -Then load the `index.html` from the project root/coverage_html dir. +### Create a micromamba/conda testing environment +> [!TIP] +> In the following instructions `micromamba` can be replaced with `conda`. + +``` +cd um2nc-standalone +micromamba env create -n um2nc_dev --file .conda/env_dev.yml +micromamba activate um2nc_dev +``` + +### Install um2nc as a development package +``` +pip install --no-deps --no-build-isolation -e . +``` + +### Running the tests + +The `um2nc-standalone` project uses `pytest` and `pytest-cov`.
+To run the tests and generate print a coverage report (with missing lines) run: + +``` +$ python3 -m pytest --cov-report=term-missing --cov=umpost +``` +> [!TIP] +> To generate an HTML coverage report substitute `term-missing` with `html`. + +## User documentation + +TODO: this needs to cover: + +1. Running `um2netcdf` standalone +2. Using the workflow run script +3. Using `um2netcdf` as an API ## Further information diff --git a/conda/meta.yaml b/conda/meta.yaml deleted file mode 100644 index a32554a..0000000 --- a/conda/meta.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %} -{% set version = data.get('version') %} -{% set pyproj = load_file_data('../pyproject.toml', from_recipe_dir=True) %} -{% set project = pyproj.get('project') %} -{% set name = project.get('name') %} - -package: - name: {{ name }} - version: "{{ version }}" - -build: - noarch: python - number: 0 - script: "python3 -m pip install . -vv" - entry_points: - {% for name, script in project.get('scripts', {}).items() %} - - {{ name }} = {{ script }} - {% endfor %} - -source: - path: ../ - -requirements: - host: - - python - - pip - - setuptools >=61.0.0 - - versioneer - run: - - python >=3.10 - {% for dep in project.get('dependencies', []) %} - - {{ dep }} - {% endfor %} - -about: - home: https://github.com/access-nri/um2nc-standalone - license: Apache Software - license_file: LICENSE - license_family: Apache - summary: "Tool to convert fieldsfiles into netCDF format. Used for post-processing UM climate model output" - diff --git a/pyproject.toml b/pyproject.toml index 9f66ef7..0ebc928 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ name = "um2nc" authors = [ {name = "Ben Davies", email="ben.davies@anu.edu.au"}, + {name = "Davide Marchegiani", email="davide.marchegiani@anu.edu.au"}, {name = "Martin Dix", email="martin.dix@anu.edu.au"}, {name = "Spencer Wong", email="spencer.wong@anu.edu.au"}, ] @@ -14,27 +15,24 @@ readme = "README.md" keywords = ["netCDF", "UM", "postprocessing"] dynamic = ["version"] dependencies = [ - "numpy", - "mule", - "cftime", - "netCDF4", - "scitools-iris", - "cf-units", - "PyYAML", - "f90nml", + "python >=3.10,<=3.12", + "cf-units", + "cftime", + "f90nml", + "mule", + "netcdf4", + "numpy", + "scitools-iris", + "versioneer", ] + [project.scripts] esm1p5_convert_nc = "umpost.conversion_driver_esm1p5:main" um2nc = "umpost.um2netcdf:main" -[tool.versioneer] -VCS = "git" -style = "pep440" -versionfile_source = "umpost/_version.py" -versionfile_build = "umpost/_version.py" -tag_prefix = "" -parentdir_prefix = "umpost-" +[project.urls] +Repository = "https://github.com/ACCESS-NRI/um2nc-standalone" [build-system] build-backend = "setuptools.build_meta" @@ -44,5 +42,13 @@ requires = [ ] [tool.setuptools.packages.find] -include = ["umpost*"] +include = ["umpost"] namespaces = false + +[tool.versioneer] +VCS = "git" +style = "pep440" +versionfile_source = "umpost/_version.py" +versionfile_build = "umpost/_version.py" +tag_prefix = "" +parentdir_prefix = "umpost-" \ No newline at end of file