Refactoring of the marine B-matrix job (NOAA-EMC#2749)

Refactor the functionality of B-matrix generation from the GDASApp

Resolves NOAA-EMC#2743

ci/cases/gfsv17/ocnanal.yaml

@@ -14,17 +14,14 @@ base:
   DO_VERFRAD: "YES"
   DO_VRFY_OCEANDA: "NO"
   FHMAX_GFS: 240
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
 ocnanal:
-  SOCA_INPUT_FIX_DIR: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca
-  CASE_ANL: 'C24'
+  SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
-  SABER_BLOCKS_YAML: ''
-  NICAS_RESOL: 1
-  NICAS_GRID_SIZE: 15000
 
 prepoceanobs:
-  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml 
+  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   OBSPREP_YAML: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
   DMPDIR: /scratch1/NCEPDEV/da/common/

env/HERA.env

@@ -77,7 +77,7 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}}
     [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max}
-    export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"    
+    export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -114,11 +114,17 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "ocnanalbmat" ]]; then
+elif [[ "${step}" = "marinebmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
 
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}"
+    export APRUN_MARINEBMAT="${launcher} -n ${npe_marinebmat}"
+
+elif [[ "${step}" = "marinebmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat}"
 
 elif [[ "${step}" = "ocnanalrun" ]]; then
 

env/HERCULES.env

@@ -112,25 +112,15 @@ case ${step} in
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;
- "ocnanalbmat")
+ "marinebmat")
 
-    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
-
-    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
-
-    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
-    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}"
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEBMAT="${launcher} -n ${npe_marinebmat}"
  ;;
  "ocnanalrun")
-
-    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
-
-    nth_max=$((npe_node_max / npe_node_ocnanalrun))
-
-    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
-    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
  ;;
 "ocnanalecen")
 

env/JET.env

@@ -102,15 +102,15 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}"
 
-elif [[ "${step}" = "ocnanalbmat" ]]; then
+elif [[ "${step}" = "marinebmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
+    nth_max=$((npe_node_max / npe_node_marinebmat))
 
-    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
+    export NTHREADS_OCNANAL=${nth_marinebmat:-${nth_max}}
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}"
+    export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat}"
 
 elif [[ "${step}" = "ocnanalrun" ]]; then
 

env/ORION.env

@@ -110,15 +110,15 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
-elif [[ "${step}" = "ocnanalbmat" ]]; then
+elif [[ "${step}" = "marinebmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
+    nth_max=$((npe_node_max / npe_node_marinebmat))
 
-    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
+    export NTHREADS_OCNANAL=${nth_marinebmat:-${nth_max}}
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}"
+    export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat} --cpus-per-task=${NTHREADS_OCNANAL}"
 
 elif [[ "${step}" = "ocnanalrun" ]]; then
 

env/S4.env

@@ -65,7 +65,7 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}}
     [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max}
-    export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc}"    
+    export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -102,10 +102,10 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}"
 
-elif [[ "${step}" = "ocnanalbmat" ]]; then
+elif [[ "${step}" = "marinebmat" ]]; then
      echo "WARNING: ${step} is not enabled on S4!"
 
-elif [[ "${step}" = "ocnanalrun" ]]; then
+elif [[ "${step}" = "marinerun" ]]; then
      echo "WARNING: ${step} is not enabled on S4!"
 
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP

@@ -28,6 +28,10 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
    COM_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL \
    COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL
 
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+   COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \
+   COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL
+
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################

jobs/JGLOBAL_ARCHIVE

@@ -35,6 +35,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 	COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \
 	COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \
 	COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+        COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \
+        COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \
 	COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \
 	COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \
 	COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \

jobs/JGLOBAL_MARINE_BMAT

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+if (( 10#${ENSMEM:-0} > 0 )); then
+  export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}"
+  export DATA="${DATAjob}/${jobid}"
+  # Create the directory to hold ensemble perturbations
+  export DATAenspert="${DATAjob}/enspert"
+  if [[ ! -d "${DATAenspert}" ]]; then mkdir -p "${DATAenspert}"; fi
+fi
+
+# source config.base, config.ocnanal and config.marinebmat
+# and pass marinebmat to ${machine}.env
+source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat"
+
+##############################################
+# Set variables used in the script
+##############################################
+# shellcheck disable=SC2153
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
+export GDUMP="gdas"
+export GDUMP_ENS="enkf${GDUMP}"
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+   COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
+   COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
+
+RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+    COMIN_OCEAN_HISTORY_ENS_PREV:COM_OCEAN_HISTORY_TMPL \
+    COMIN_ICE_HISTORY_ENS_PREV:COM_ICE_HISTORY_TMPL
+
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+   COMOUT_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \
+   COMOUT_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL
+
+mkdir -p "${COMOUT_OCEAN_BMATRIX}"
+mkdir -p "${COMOUT_ICE_BMATRIX}"
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASMARINEBMATRUNPY:-${SCRgfs}/exglobal_marinebmat.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/rocoto/ocnanalbmat.sh → jobs/rocoto/marinebmat.sh

@@ -8,12 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 [[ "${status}" -ne 0 ]] && exit "${status}"
 
-export job="ocnanalbmat"
+export job="marinebmat"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT
-echo "BMAT gets run here"
+"${HOMEgfs}"/jobs/JGLOBAL_MARINE_BMAT
 status=$?
 exit "${status}"

parm/archive/gdas.yaml.j2

@@ -22,7 +22,7 @@ gdas:
             {% if DO_JEDIOCNVAR %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}prepoceanobs.log"
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalprep.log"
-        - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalbmat.log"
+        - "logs/{{ cycle_YMDH }}/{{ RUN }}marinebmat.log"
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalrun.log"
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalpost.log"
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalchkpt.log"

parm/archive/gdasocean_analysis.yaml.j2

@@ -3,23 +3,30 @@ gdasocean_analysis:
     name: "GDASOCEAN_ANALYSIS"
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar"
     required:
+        # analysis and analysis increments
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc'
-        {% set ocngrid_cycle = '%02d' % (((cycle_HH | int) - 3) % 24) %}
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc'
         {% for domain in ["ocn", "ice"] %}
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.bkgerr_stddev.nc'
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc'
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc'
-        {% if NMEM_ENS > 2 %}
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc'
-        {% endif %}
         {% endfor %}
+
+        # static background error
+        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.bkgerr_stddev.nc'
+        - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ice.bkgerr_stddev.nc'
+
+        # ensemble background error
         {% if NMEM_ENS > 2 %}
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc'
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc'
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc'
-        - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc'
+        - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ice.ens_weights.nc'
+        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.ens_weights.nc'
+        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.recentering_error.nc'
+        {% for diag_type in ["ssh_steric_stddev", "ssh_unbal_stddev", "ssh_total_stddev", "steric_explained_variance"] %}
+        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.{{ diag_type }}.nc'
+        {% endfor %}
         {% endif %}
+
+        # obs space diags
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.*.stats.csv'
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4'
+
+        # runtime configs
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml'