update develop branch

.cicd/Jenkinsfile

@@ -137,16 +137,6 @@ pipeline {
                         }
                     }
 
-                    // Run the unittest functional tests that require an HPC platform
-                    stage('Functional UnitTests') {
-                        steps {
-                            dir ("${env.SRW_PLATFORM}") {
-                            echo "Running unittest on retrieve_data.py"
-                            sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"'
-                            }
-                        }
-                    }
-
                     // Run the unified build script; if successful create a tarball of the build and upload to S3
                     stage('Build') {
                         steps {
@@ -164,6 +154,16 @@ pipeline {
                         }
                     }
 
+                    // Run the unittest functional tests that require an HPC platform
+                    stage('Functional UnitTests') {
+                        steps {
+                            dir ("${env.SRW_PLATFORM}") {
+                            echo "Running unittest on retrieve_data.py"
+                            sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"'
+                            }
+                        }
+                    }
+
                     // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage
                     stage('Functional WorkflowTaskTests') {
                         steps {

.cicd/scripts/srw_ftest.sh

@@ -85,11 +85,9 @@ module load build_${platform,,}_${SRW_COMPILER}
 module load wflow_${platform,,}
 
 [[ ${FORGIVE_CONDA} == true ]] && set +e +u    # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests
-conda activate workflow_tools
+conda activate srw_app
 set -e -u
 
-export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src
-
 # Adjust for strict limitation of stack size 
 sed "s|ulimit -s unlimited;|ulimit -S -s unlimited;|" -i ${workspace}/ush/machine/hera.yaml
 
@@ -102,7 +100,7 @@ cd ${workspace}
 
 cd ${EXPTDIR}
 pwd
-cp ${workspace}/ush/wrappers/* .
+cp ${workspace}/ush/wrappers/*.sh .
 
 # Set parameters that the task scripts require ...
 export JOBSdir=${workspace}/jobs

.github/workflows/python_tests.yaml

@@ -0,0 +1,58 @@
+name: Python Tests
+on:
+  push:
+    branches:
+      - develop
+      - 'release/*'
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+jobs:
+  srw_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Micromamba and srw_app environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          environment-name: srw_app
+          init-shell: bash
+          cache-downloads: true
+          cache-environment: true
+
+      - name: Checkout externals
+        run: |
+          ./manage_externals/checkout_externals ufs-weather-model
+
+      - name: Lint the python code
+        run: |
+          micromamba activate srw_app
+          export PYTHONPATH=$(pwd)/ush
+          pylint --ignore-imports=yes tests/test_python/
+          pylint ush/create_*.py
+          pylint ush/generate_FV3LAM_wflow.py
+
+      - name: Run python unittests
+        run: |
+          # exclude test_retrieve_data that is tested in functional test
+          micromamba activate srw_app
+          export UNIT_TEST=True
+          export PYTHONPATH=$(pwd)/ush
+          python -m unittest -b tests/test_python/*.py
+
+      - name: Run python functional tests
+        run: |
+          micromamba activate srw_app
+          export CI=true
+          export PYTHONPATH=${PWD}/ush
+          python3 -m unittest -b tests/test_python/test_retrieve_data.py

.gitignore

@@ -15,6 +15,7 @@ ush/config.yaml
 ush/python_utils/__pycache__/
 ush/python_utils/workflow-tools/
 ush/*.swp
+conda_loc
 
 *.swp
 __pycache__

.pylintrc

@@ -182,6 +182,7 @@ function-naming-style=snake_case
 good-names=i,
            j,
            k,
+           e,
            ex,
            Run,
            _
@@ -422,7 +423,9 @@ disable=raw-checker-failed,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        logging-fstring-interpolation
+        logging-fstring-interpolation,
+        too-many-locals,
+        similarities
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

Externals.cfg

@@ -43,14 +43,5 @@ hash = 694a139
 local_path = sorc/AQM-utils
 required = True
 
-[workflow-tools]
-protocol = git
-repo_url = https://github.com/ufs-community/workflow-tools
-# Specify either a branch name or a hash but not both.
-# branch = develop
-hash = e1b3b6f
-local_path = ush/python_utils/workflow-tools
-required = True
-
 [externals_description]
 schema_version = 1.0.0

README.md

@@ -1,18 +1,18 @@
 # UFS Short-Range Weather Application
 
-The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA’s operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader Weather Enterprise. For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
+The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. NOAA's operational model suite for numerical weather prediction (NWP) is quickly transitioning to the UFS from a number of legacy modeling systems. The UFS enables research, development, and contribution opportunities within the broader Weather Enterprise (including government, industry, and academia). For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
 
-The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.1.0) represents a snapshot of this continuously evolving system. 
+The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.2.0) represents a snapshot of this continuously evolving system. 
 
-The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.1.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.1.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
+The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.2.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.2.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
 
 For instructions on how to clone the repository, build the code, and run the workflow, see:
-https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started
+- https://ufs-srweather-app.readthedocs.io/en/develop/BuildingRunningTesting/Quickstart.html
 
 For a debugging guide for users and developers in the field of Earth System Modeling, please see:
 https://epic.noaa.gov/wp-content/uploads/2022/12/Debugging-Guide.pdf
 
-UFS Development Team. (2022, Nov. 17). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.1.0). Zenodo. https://doi.org/10.5281/zenodo.7277602
+UFS Development Team. (2023, Oct. 31). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.2.0). Zenodo. https://doi.org/10.5281/zenodo.10015544
 
 [![Python unittests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml)
 [![Python functional tests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml)

aqm_environment.yml

@@ -0,0 +1,13 @@
+name: srw_aqm
+channels:
+  - conda-forge
+  - ufs-community
+dependencies:
+  - esmpy
+  - netcdf4=1.6.*
+  - numpy=1.23.*
+  - pylint=2.17*
+  - pytest=7.2*
+  - scipy=1.10.*
+  - uwtools=1.0.0
+  - xarray=2022.11.*