Skip to content

Commit

Permalink
Add snow DA update and recentering for the EnKF forecasts (NOAA-EMC#2690
Browse files Browse the repository at this point in the history
)

This PR adds the capability to update the ensemble of snow states 
by recentering the ensemble mean to the deterministic snow analysis
and applying increments as appropriate.

Resolves NOAA-EMC#2585

---------

Co-authored-by: David Huber <[email protected]>
Co-authored-by: Rahul Mahajan <[email protected]>
Co-authored-by: Guillaume Vernieres <[email protected]>
Co-authored-by: AntonMFernando-NOAA <[email protected]>
Co-authored-by: Anil Kumar <[email protected]>
Co-authored-by: TerrenceMcGuinness-NOAA <[email protected]>
  • Loading branch information
7 people authored Aug 23, 2024
1 parent 1b18f2f commit 2ce2116
Show file tree
Hide file tree
Showing 29 changed files with 911 additions and 42 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ ush/month_name.sh
ush/imsfv3_scf2ioda.py
ush/atparse.bash
ush/run_bufr2ioda.py
ush/bufr2ioda_insitu*

# version files
versions/build.ver
Expand Down
4 changes: 2 additions & 2 deletions ci/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ pipeline {
}
}
}


stage( '5. FINALIZE' ) {
agent { label NodeName[machine].toLowerCase() }
Expand Down Expand Up @@ -297,6 +297,6 @@ pipeline {
}
}
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ arguments:
pslot: {{ 'pslot' | getenv }}
app: ATMA
resdetatmos: 96
resensatmos: 48
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
idate: 2021122012
edate: 2021122100
nens: 0
nens: 2
gfs_cyc: 1
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
Expand Down
1 change: 1 addition & 0 deletions ci/cases/pr/C96C48_ufs_hybatmDA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ skip_ci_on_hosts:
- gaea
- orion
- hercules
- wcoss2

9 changes: 8 additions & 1 deletion env/HERA.env
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
elif [[ "${step}" = "atmensanlfv3inc" ]]; then

export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"

elif [[ "${step}" = "aeroanlrun" ]]; then

Expand All @@ -106,6 +106,13 @@ elif [[ "${step}" = "snowanl" ]]; then

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "esnowrecen" ]]; then

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "marinebmat" ]]; then

export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
Expand Down
8 changes: 8 additions & 0 deletions env/HERCULES.env
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,14 @@ case ${step} in

export APRUN_APPLY_INCR="${launcher} -n 6"
;;
"esnowrecen")

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"
;;

"marinebmat")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
Expand Down
7 changes: 7 additions & 0 deletions env/JET.env
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "esnowrecen" ]]; then

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "atmanlfv3inc" ]]; then

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
Expand Down
7 changes: 7 additions & 0 deletions env/ORION.env
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ elif [[ "${step}" = "snowanl" ]]; then

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "esnowrecen" ]]; then

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "atmanlfv3inc" ]]; then

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
Expand Down
9 changes: 8 additions & 1 deletion env/S4.env
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
elif [[ "${step}" = "atmensanlfv3inc" ]]; then

export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
export APRUN_ATMENSANLFV3INC="${APRUN}"
export APRUN_ATMENSANLFV3INC="${APRUN}"

elif [[ "${step}" = "aeroanlrun" ]]; then

Expand All @@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "esnowrecen" ]]; then

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "atmanlfv3inc" ]]; then

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
Expand Down
7 changes: 7 additions & 0 deletions env/WCOSS2.env
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,13 @@ elif [[ "${step}" = "snowanl" ]]; then

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "esnowrecen" ]]; then

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN}"

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "atmanlfv3inc" ]]; then

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
Expand Down
59 changes: 59 additions & 0 deletions jobs/JGDAS_ENKF_SNOW_RECENTER
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#! /usr/bin/env bash

source "${HOMEgfs}/ush/preamble.sh"
source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowrecen" -c "base esnowrecen"

##############################################
# Set variables used in the script
##############################################
# Ignore possible spelling error (nothing is misspelled)
# shellcheck disable=SC2153
GDUMP="gdas"
export GDUMP

##############################################
# Begin JOB SPECIFIC work
##############################################
# Generate COM variables from templates
YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
COMIN_OBS:COM_OBS_TMPL \
COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
COMOUT_CONF:COM_CONF_TMPL
MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl \
COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL

mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"

for imem in $(seq 1 "${NMEM_ENS}"); do
memchar="mem$(printf %03i "${imem}")"
MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
mkdir -p "${COMOUT_SNOW_ANALYSIS}"
done

###############################################################
# Run relevant script

EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exgdas_enkf_snow_recenter.py}
${EXSCRIPT}
status=$?
(( status != 0 )) && exit "${status}"

##############################################
# End JOB SPECIFIC work
##############################################

##############################################
# Final processing
##############################################
if [[ -e "${pgmout}" ]] ; then
cat "${pgmout}"
fi

##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" || exit 1
[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"

exit 0
6 changes: 6 additions & 0 deletions jobs/JGLOBAL_PREP_SNOW_OBS
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,10 @@ if [[ -e "${pgmout}" ]] ; then
cat "${pgmout}"
fi

##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" || exit 1
[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"

exit 0
6 changes: 6 additions & 0 deletions jobs/JGLOBAL_SNOW_ANALYSIS
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,10 @@ if [[ -e "${pgmout}" ]] ; then
cat "${pgmout}"
fi

##########################################
# Remove the Temporary working directory
##########################################
cd "${DATAROOT}" || exit 1
[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"

exit 0
18 changes: 18 additions & 0 deletions jobs/rocoto/esnowrecen.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#! /usr/bin/env bash

source "${HOMEgfs}/ush/preamble.sh"

###############################################################
# Source UFSDA workflow modules
. "${HOMEgfs}/ush/load_ufsda_modules.sh"
status=$?
[[ ${status} -ne 0 ]] && exit "${status}"

export job="esnowrecen"
export jobid="${job}.$$"

###############################################################
# Execute the JJOB
"${HOMEgfs}/jobs/JGDAS_ENKF_SNOW_RECENTER"
status=$?
exit "${status}"
29 changes: 29 additions & 0 deletions parm/config/gfs/config.esnowrecen
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#! /usr/bin/env bash

########## config.esnowrecen ##########
# configuration common to snow ensemble analysis tasks

echo "BEGIN: config.esnowrecen"

# Get task specific resources
source "${EXPDIR}/config.resources" esnowrecen

export JCB_BASE_YAML="${PARMgfs}/gdas/snow/jcb-base.yaml.j2"
export JCB_ALGO_YAML="${PARMgfs}/gdas/snow/jcb-fv3jedi_land_ensrecenter.yaml.j2"

export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
export SNOW_ENS_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_ens_update.yaml.j2"
export SNOW_OROG_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_orog.yaml.j2"
export SNOW_ENS_FINALIZE_TMPL="${PARMgfs}/gdas/snow_finalize_ens_update.yaml.j2"

# Name of the executable that applies increment to bkg and its namelist template
export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/ens_apply_incr_nml.j2"

export io_layout_x=@IO_LAYOUT_X@
export io_layout_y=@IO_LAYOUT_Y@

export JEDIEXE=${EXECgfs}/gdasapp_land_ensrecenter.x
export FREGRID=${EXECgfs}/fregrid.x

echo "END: config.esnowrecen"
31 changes: 30 additions & 1 deletion parm/config/gfs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ if (( $# != 1 )); then
echo "prep prepsnowobs prepatmiodaobs"
echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal"
echo "snowanl"
echo "snowanl esnowrecen"
echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal"
echo "anal sfcanl analcalc analdiag fcst echgres"
echo "upp atmos_products"
Expand Down Expand Up @@ -348,6 +348,35 @@ case ${step} in
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
;;

"esnowrecen")
# below lines are for creating JEDI YAML
case ${CASE} in
"C768")
layout_x=6
layout_y=6
;;
"C384")
layout_x=5
layout_y=5
;;
"C192" | "C96" | "C48")
layout_x=1
layout_y=1
;;
*)
echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
exit 4
esac

export layout_x
export layout_y

walltime="00:15:00"
ntasks=$(( layout_x * layout_y * 6 ))
threads_per_task=1
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
;;

"prepobsaero")
walltime="00:30:00"
ntasks=1
Expand Down
43 changes: 43 additions & 0 deletions parm/gdas/snow_finalize_ens_update.yaml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
copy:
######################################
# copy analyses to directories
######################################
{% for mem in range(1, NMEM_ENS + 1) %}
# define variables
# Declare a dict of search and replace terms to run on each template
{% set tmpl_dict = {'${ROTDIR}':ROTDIR,
'${RUN}':RUN,
'${YMD}':current_cycle | to_YMD ,
'${HH}':current_cycle | strftime("%H"),
'${MEMDIR}':"mem" + '%03d' % mem} %}

{% for tile in range(1, ntiles+1) %}
- ["{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
{% endfor %}
{% if DOIAU == True %}
# if using IAU, also need analyses copied at the beginning of the window
{% for tile in range(1, ntiles+1) %}
- ["{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
{% endfor %}
{% endif %}
{% endfor %}
######################################
# copy ensemble mean increment to COM
######################################
# define variables
# Declare a dict of search and replace terms to run on each template
{% set tmpl_dict = {'${ROTDIR}':ROTDIR,
'${RUN}':RUN,
'${YMD}':current_cycle | to_YMD ,
'${HH}':current_cycle | strftime("%H"),
'${MEMDIR}':"ensstat"} %}

{% for tile in range(1, ntiles+1) %}
- ["{{ DATA }}/inc/ensmean/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
{% endfor %}
{% if DOIAU == True %}
# if using IAU, also need increment copied at the beginning of the window
{% for tile in range(1, ntiles+1) %}
- ["{{ DATA }}/inc/ensmean/snowinc.{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
{% endfor %}
{% endif %}
Loading

0 comments on commit 2ce2116

Please sign in to comment.