Switch snow DA to use 2DVar for deterministic and ensemble mean (#3163)

This PR moves the snow DA from a LETKF-OI approach with a fake ensemble to a 2DVar approach for the deterministic and the ensemble mean for GDAS/GFS. This PR also adds support for JCB and refactoring of the jobs to leverage the Jedi class in the workflow. Resolves #3002 --------- Co-authored-by: RussTreadon-NOAA <[email protected]> Co-authored-by: Jiarui Dong <[email protected]> Co-authored-by: DavidNew-NOAA <[email protected]> Co-authored-by: Cory Martin <[email protected]> Co-authored-by: Walter Kolczynski - NOAA <[email protected]>
NOAA-EMC · Dec 20, 2024 · d479111 · d479111
1 parent 2007864
commit d479111
Show file tree

Hide file tree

Showing 35 changed files with 782 additions and 835 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -115,14 +115,16 @@ elif [[ "${step}" = "snowanl" ]]; then
     export APRUN_CALCFIMS="${launcher} -n 1"
 
     export NTHREADS_SNOWANL=${NTHREADSmax}
-    export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}"
+    export APRUN_SNOWANL="${APRUN_default} --mem=0 --cpus-per-task=${NTHREADS_SNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "esnowrecen" ]]; then
+elif [[ "${step}" = "esnowanl" ]]; then
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default} --mem=0 --cpus-per-task=${NTHREADS_ESNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -118,10 +118,12 @@ case ${step} in
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;
- "esnowrecen")
+ "esnowanl")
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;

diff --git a/env/JET.env b/env/JET.env
@@ -102,10 +102,12 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "esnowrecen" ]]; then
+elif [[ "${step}" = "esnowanl" ]]; then
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

diff --git a/env/ORION.env b/env/ORION.env
@@ -109,10 +109,12 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "esnowrecen" ]]; then
+elif [[ "${step}" = "esnowanl" ]]; then
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

diff --git a/env/S4.env b/env/S4.env
@@ -102,10 +102,12 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "esnowrecen" ]]; then
+elif [[ "${step}" = "esnowanl" ]]; then
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -95,10 +95,12 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "esnowrecen" ]]; then
+elif [[ "${step}" = "esnowanl" ]]; then
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 

diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE
@@ -10,7 +10,8 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "earc" -c "base earc"
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP
 MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
   COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
-  COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL
+  COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \
+  COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL
 
 ###############################################################
 # Run archive script

diff --git a/jobs/JGDAS_ENKF_SNOW_RECENTER → jobs/JGLOBAL_SNOWENS_ANALYSIS b/jobs/JGDAS_ENKF_SNOW_RECENTER → jobs/JGLOBAL_SNOWENS_ANALYSIS
@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowrecen" -c "base esnowrecen"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowanl" -c "base esnowanl"
 
 ##############################################
 # Set variables used in the script
@@ -10,19 +10,18 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowrecen" -c "base esnowrecen"
 # shellcheck disable=SC2153
 GDUMP="gdas"
 export GDUMP
+CDUMP=${RUN/enkf}
+export CDUMP
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 # Generate COM variables from templates
+RUN=${CDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_OBS:COM_OBS_TMPL
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
-    COMIN_OBS:COM_OBS_TMPL \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
     COMOUT_CONF:COM_CONF_TMPL
-MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl \
-    COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
-
-mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"
 
 for imem in $(seq 1 "${NMEM_ENS}"); do
     memchar="mem$(printf %03i "${imem}")"
@@ -31,10 +30,15 @@ for imem in $(seq 1 "${NMEM_ENS}"); do
     mkdir -p "${COMOUT_SNOW_ANALYSIS}"
 done
 
+MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -x\
+    COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+
+mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"
+
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exgdas_enkf_snow_recenter.py}
+EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exglobal_snowens_analysis.py}
 ${EXSCRIPT}
 status=$?
 (( status != 0 )) && exit "${status}"

diff --git a/jobs/JGLOBAL_SNOW_ANALYSIS b/jobs/JGLOBAL_SNOW_ANALYSIS
@@ -1,7 +1,6 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-export DATA=${DATA:-${DATAROOT}/${RUN}snowanl_${cyc}}
 source "${HOMEgfs}/ush/jjob_header.sh" -e "snowanl" -c "base snowanl"
 
 ##############################################
@@ -18,12 +17,15 @@ GDUMP="gdas"
 # Begin JOB SPECIFIC work
 ##############################################
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_SNOW_ANALYSIS COM_CONF
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
+    COMOUT_CONF:COM_CONF_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+    COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
 
-mkdir -m 775 -p "${COM_SNOW_ANALYSIS}" "${COM_CONF}"
+mkdir -m 775 -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/rocoto/esnowanl.sh b/jobs/rocoto/esnowanl.sh
@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="esnowanl"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for ioda utilities
+# shellcheck disable=SC2311
+pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
+gdasappPATH="${HOMEgfs}/sorc/gdas.cd/sorc/iodaconv/src:${pyiodaPATH}"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${gdasappPATH}"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_SNOWENS_ANALYSIS"
+status=$?
+exit "${status}"
diff --git a/jobs/rocoto/esnowrecen.sh b/jobs/rocoto/esnowrecen.sh
diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2
@@ -70,6 +70,14 @@ enkf:
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}"
         {% endfor %}
 
+        {% if DO_JEDISNOWDA %}
+        - "{{ COMIN_SNOW_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}snowstat.tgz"
+        {% for itile in range(1,7) %}
+        # Snow analysis is 3dvar
+        - "{{ COMIN_SNOW_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc"
+        {% endfor %}
+        {% endif %}
+
         # Ensemble mean analyses/increments
         # 6-hr analysis/increment
         {% if do_calc_increment %}

diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2
@@ -47,7 +47,7 @@ gdas_restarta:
 
         # Snow configuration yaml
         {% if DO_JEDISNOWDA %}
-        - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml"
+        - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}snowanlvar.yaml"
         {% endif %}
 
         # Input BUFR files

diff --git a/parm/config/gfs/config.esnowanl b/parm/config/gfs/config.esnowanl
@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+
+########## config.esnowanl ##########
+# configuration common to snow ensemble analysis tasks
+
+echo "BEGIN: config.esnowanl"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" esnowanl
+
+export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2"
+export GTS_SNOW_STAGE_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2"
+
+export JCB_BASE_YAML="${PARMgfs}/gdas/snow/jcb-base.yaml.j2"
+export JCB_ALGO_YAML_VAR="${PARMgfs}/gdas/snow/jcb-prototype_2dvar.yaml.j2"
+
+# Process IMS snowcover into snow depth
+export IMS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_ims.yaml.j2"
+export CALCFIMSEXE="${EXECgfs}/calcfIMS.exe"
+export FIMS_NML_TMPL="${PARMgfs}/gdas/snow/prep/fims.nml.j2"
+export IMS2IODACONV="${USHgfs}/imsfv3_scf2ioda.py"
+
+export JEDI_FIX_YAML="${PARMgfs}/gdas/snow_jedi_fix.yaml.j2"
+export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/snow_berror.yaml.j2"
+export SNOW_ENS_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_ens_update.yaml.j2"
+export SNOW_OROG_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_orog.yaml.j2"
+export SNOW_ENS_FINALIZE_TMPL="${PARMgfs}/gdas/snow_finalize_ens_update.yaml.j2"
+
+# Name of the executable that applies increment to bkg and its namelist template
+export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
+export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2"
+
+export JEDI_CONFIG_YAML="${PARMgfs}/gdas/esnowanl_jedi_config.yaml.j2"
+
+export io_layout_x=@IO_LAYOUT_X@
+export io_layout_y=@IO_LAYOUT_Y@
+
+echo "END: config.esnowanl"
diff --git a/parm/config/gfs/config.esnowrecen b/parm/config/gfs/config.esnowrecen
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -15,7 +15,7 @@ if (( $# != 1 )); then
     echo "prep prepatmiodaobs"
     echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
     echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal"
-    echo "snowanl esnowrecen"
+    echo "snowanl esnowanl"
     echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
     echo "anal sfcanl analcalc analdiag fcst echgres"
     echo "upp atmos_products"
@@ -346,11 +346,11 @@ case ${step} in
 
     walltime="00:15:00"
     ntasks=$(( layout_x * layout_y * 6 ))
-    threads_per_task=1
+    threads_per_task=2
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
-  "esnowrecen")
+  "esnowanl")
     # below lines are for creating JEDI YAML
     case ${CASE} in
       "C768")
@@ -373,9 +373,9 @@ case ${step} in
     export layout_x
     export layout_y
 
-    walltime="00:15:00"
+    walltime="00:30:00"
     ntasks=$(( layout_x * layout_y * 6 ))
-    threads_per_task=1
+    threads_per_task=2
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
@@ -1213,7 +1213,7 @@ case ${step} in
     ;;
 
   "esfc")
-    walltime="00:15:00"
+    walltime="00:25:00"
     ntasks=80
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))