SFS on AWS

SFS_baseline-c96/SFS.yaml

@@ -0,0 +1,22 @@
+experiment:
+  system: gefs
+  mode: forecast-only
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 96
+  resensatmos: 96
+  resdetocean: 1.0
+  nens: 10
+  gfs_cyc: 1
+  start: cold
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 1994050100
+  edate: 1994051212
+  yaml: {{ HOMEgfs }}/SFS_baseline-c96/SFS_options.yaml
+
+skip_ci_on_hosts:
+  - wcoss2
+

SFS_baseline-c96/SFS_options.yaml

@@ -0,0 +1,35 @@
+base:
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "NO"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  KEEPDATA: "YES"
+  DO_EXTRACTVARS: "NO"
+  FHMAX_GFS: 600
+  FHMAX_HF_GFS: 0
+  FHOUT_HF_GFS: 1
+  FHOUT_OCN_GFS: 24
+  FHOUT_ICE_GFS: 24
+  FCST_BREAKPOINTS: "300"
+  REPLAY_ICS: "NO"
+  HPSSARCH: "NO"
+  LOCALARCH: "NO"
+  FLTFILEGFS: "postxconfig-NT-SFS.txt"
+  FLTFILEGFSF00: "postxconfig-NT-SFS.txt"
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  BASE_IC: /bucket/Wei.Huang/SFS-REPLAY_ICs/C96mx100
+fcst:
+  TYPE: "hydro"
+  MONO: "mono"
+  reforecast: "YES"
+  FHZER: 12
+  POST_SFS: "YES"
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
+ocn:
+  MOM6_INTERP_ICS: "YES"

env/AWSPW.env

@@ -43,6 +43,11 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${APRUN_default}"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

parm/config/gefs/config.base

@@ -260,8 +260,8 @@ export FHOUT_OCN=${FHOUT_OCN_GFS}
 export FHOUT_ICE=${FHOUT_ICE_GFS}
 
 # GFS restart interval in hours
-export restart_interval_gfs=12
-export restart_interval_enkfgfs=12
+export restart_interval_gfs=600
+export restart_interval_enkfgfs=600
 # NOTE: Do not set this to zero.  Instead set it to $FHMAX_GFS
 # TODO: Remove this variable from config.base and reference from config.fcst
 # TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used

parm/config/gefs/config.fcst

@@ -53,9 +53,11 @@ if [[ "${DONST}" == "YES" ]]; then
     source "${EXPDIR}/config.nsst"
 fi
 
-export esmf_profile=".false."
-export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE
+#export esmf_profile=".false."
+#export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE
 
+export esmf_profile=".true."
+export esmf_logkind="ESMF_LOGKIND_MULTI" #Options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE
 
 #######################################################################
 

parm/config/gefs/config.resources

@@ -41,7 +41,9 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    max_tasks_per_node=36
+    max_tasks_per_node=24
+   #max_tasks_per_node=36
+   #max_tasks_per_node=96
     ;;
   "AZUREPW")
     export PARTITION_BATCH="compute"
@@ -188,7 +190,7 @@ case ${step} in
 
     case "${CASE}" in
       "C48" | "C96" | "C192")
-        declare -x "walltime"="03:00:00"
+        declare -x "walltime"="06:00:00"
         ;;
       "C384" | "C768" | "C1152")
         declare -x "walltime"="06:00:00"

parm/config/gefs/config.ufs

@@ -316,7 +316,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       DT_THERM_MOM6='3600'
       FRUNOFF=""
       CHLCLIM="seawifs_1998-2006_smoothed_2X.nc"
-      MOM6_RESTART_SETTING='r'
+      MOM6_RESTART_SETTING='n'
       MOM6_RIVER_RUNOFF='False'
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_MISVAL="-1e34"
@@ -337,7 +337,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       DT_THERM_MOM6='3600'
       FRUNOFF="runoff.daitren.clim.1deg.nc"
       CHLCLIM="seawifs_1998-2006_smoothed_2X.nc"
-      MOM6_RESTART_SETTING='r'
+      MOM6_RESTART_SETTING='n'
       MOM6_RIVER_RUNOFF='False'
       eps_imesh="2.5e-1"
       TOPOEDITS="ufs.topo_edits_011818.nc"
@@ -382,7 +382,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc"
       CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc"
       MOM6_RIVER_RUNOFF='True'
-      MOM6_RESTART_SETTING="r"
+      MOM6_RESTART_SETTING="n"
       eps_imesh="1.0e-1"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"

parm/config/gefs/yaml/SFS_defaults.yaml

@@ -0,0 +1,32 @@
+base:
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "NO"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  KEEPDATA: "NO"
+  DO_EXTRACTVARS: "NO"
+ #FHMAX_GFS: 2976
+  FHMAX_GFS: 120
+  FHMAX_HF_GFS: 0
+  FHOUT_HF_GFS: 1
+  FHOUT_OCN_GFS: 24
+  FHOUT_ICE_GFS: 24
+  FCST_BREAKPOINTS: ""
+  REPLAY_ICS: "NO"
+  HPSSARCH: "NO"
+  LOCALARCH: "NO"
+fcst:
+  TYPE: "hydro"
+  MONO: "mono"
+  reforecast: "YES"
+  FHZER: 12
+  POST_SFS: "YES"
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
+ocn:
+  MOM6_INTERP_ICS: "YES"

parm/config/gfs/config.resources

@@ -107,8 +107,10 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    npe_node_max=36
-    max_tasks_per_node=36
+   #npe_node_max=36
+   #max_tasks_per_node=36
+    npe_node_max=96
+    max_tasks_per_node=96
     # TODO Supply a max mem/node value for AWS
     # shellcheck disable=SC2034
     mem_node_max=""

scripts/exglobal_forecast.sh

@@ -149,6 +149,11 @@ else
   export OMP_NUM_THREADS=${UFS_THREADS:-1}
 fi
 
+#module load slurm
+
+export I_MPI_FABRICS=shm:ofi
+export I_MPI_OFI_PROVIDER=tcp
+
 ${NCP} "${EXECgfs}/${FCSTEXEC}" "${DATA}/"
 ${APRUN_UFS} "${DATA}/${FCSTEXEC}" 1>&1 2>&2
 export ERR=$?

sfsc96.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ set -x
+
+ source ~/.bashrc
+ source workflow/gw_setup.sh
+
+ HPC_ACCOUNT=${USER} \
+	IDATE=1994052600 \
+	pslot=c96sfs \
+	RUNTESTS=/lustre/$USER/run \
+	./workflow/create_experiment.py \
+        --yaml SFS_baseline-c96/SFS.yaml
+

ush/parsing_namelists_MOM6.sh

@@ -6,6 +6,7 @@ MOM6_namelists(){
 
 # MOM6 namelists generation
 
+MOM6_RESTART_SETTING=n
 # ================================================================
 # input.nml
 # ---------

workflow/hosts/awspw.yaml

@@ -1,7 +1,7 @@
 BASE_GIT: '' #TODO: This does not yet exist.
 DMPDIR: '' # TODO: This does not yet exist.
 PACKAGEROOT: '' #TODO: This does not yet exist.
-COMINsyn: '' #TODO: This does not yet exist.
+COMINsyn: '/contrib/global-workflow-shared-data/gfs/prod/syndat' #TODO: This does not yet exist.
 HOMEDIR: '/contrib/${USER}'
 STMP: '/lustre/${USER}/stmp/'
 PTMP: '/lustre/${USER}/ptmp/'
@@ -18,10 +18,11 @@ CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
-BASE_DATA: '/bucket/global-workflow-shared-data'
+BASE_DATA: '/bucket/global-workflow-shared-data/data'
 BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR'
+#BASE_IC: '/bucket/Wei.Huang/SFS-REPLAY_ICs/C96mx100'
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from AWS.
 MAKE_NSSTBUFR: 'NO'
 MAKE_ACFTBUFR: 'NO'
-SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions.
+SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C768'] # TODO: Test and support all cubed-sphere resolutions.