From 1c5b14bea76609bfff317be1c300562a4fb7762b Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 13 Nov 2024 21:39:32 -0500
Subject: [PATCH 01/37] allow building multiple ufs_model.x for gfs, gefs and
 sfs

---
 parm/config/gefs/config.fcst |  6 +++---
 parm/config/gfs/config.fcst  |  6 +++---
 sorc/build_all.sh            |  7 ++++---
 sorc/build_ufs.sh            | 20 ++++++++++++++------
 4 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst
index c600c8edbf..21c666c520 100644
--- a/parm/config/gefs/config.fcst
+++ b/parm/config/gefs/config.fcst
@@ -63,7 +63,7 @@ export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_O
 
 export FORECASTSH="${SCRgfs}/exglobal_forecast.sh"
 #export FORECASTSH="${SCRgfs}/exglobal_forecast.py"  # Temp. while this is worked on
-export FCSTEXEC="ufs_model.x"
+export FCSTEXEC="gefs_model.x"
 
 #######################################################################
 # Model configuration
@@ -117,7 +117,7 @@ if (( gwd_opt == 2 )); then
     export do_gsl_drag_ls_bl=".true."
     export do_gsl_drag_ss=".false."
     export do_gsl_drag_tofd=".true."
-    export do_gwd_opt_psl=".true."    
+    export do_gwd_opt_psl=".true."
     export do_ugwp_v1_orog_only=".false."
     launch_level=$(echo "${LEVS}/2.35" |bc)
     export launch_level
@@ -145,7 +145,7 @@ if [[ "${progsigma}" == ".true." ]]; then tbp="_progsigma" ; fi
 if [[ "${DO_AERO}" == "YES" ]]; then
     export IAER=2011  # spectral band mapping method for aerosol optical properties
 else
-    export IAER=1011    
+    export IAER=1011
 fi
 export iovr_lw=3    ; #de-correlation length cloud overlap method (Barker, 2008)
 export iovr_sw=3    ; #de-correlation length cloud overlap method (Barker, 2008)
diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst
index 571e6cafb5..7fca6ac0ca 100644
--- a/parm/config/gfs/config.fcst
+++ b/parm/config/gfs/config.fcst
@@ -74,7 +74,7 @@ export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_O
 
 export FORECASTSH="${SCRgfs}/exglobal_forecast.sh"
 #export FORECASTSH="${SCRgfs}/exglobal_forecast.py"  # Temp. while this is worked on
-export FCSTEXEC="ufs_model.x"
+export FCSTEXEC="gfs_model.x"
 
 #######################################################################
 # Model configuration
@@ -128,7 +128,7 @@ if (( gwd_opt == 2 )); then
     export do_gsl_drag_ls_bl=".true."
     export do_gsl_drag_ss=".false."
     export do_gsl_drag_tofd=".true."
-    export do_gwd_opt_psl=".true."    
+    export do_gwd_opt_psl=".true."
     export do_ugwp_v1_orog_only=".false."
     launch_level=$(echo "${LEVS}/2.35" |bc)
     export launch_level
@@ -160,7 +160,7 @@ if [[ "${progsigma}" == ".true." ]]; then tbp="_progsigma" ; fi
 if [[ "${DO_AERO}" == "YES" ]]; then
     export IAER=2011  # spectral band mapping method for aerosol optical properties
 else
-    export IAER=1011    
+    export IAER=1011
 fi
 export iovr_lw=3    ; #de-correlation length cloud overlap method (Barker, 2008)
 export iovr_sw=3    ; #de-correlation length cloud overlap method (Barker, 2008)
diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index e75c853c39..9414846f2a 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -56,6 +56,7 @@ _wave_opt=""
 _hydro_opt=""
 _build_job_max=20
 _quick_kill="NO"
+_ufs_exec="-e gfs_model.x"
 # Reset option counter in case this script is sourced
 OPTIND=1
 while getopts ":a:dfghj:kuvwy" option; do
@@ -69,8 +70,8 @@ while getopts ":a:dfghj:kuvwy" option; do
     k) _quick_kill="YES" ;;
     u) _build_ufsda="YES" ;;
     v) _verbose_opt="-v";;
-    w) _wave_opt="-w";;
-    y) _hydro_opt="-y";;
+    w) _wave_opt="-w"; _ufs_exec="-e gefs_model.x";;
+    y) _hydro_opt="-y"; _ufs_exec="-e sfs_model.x";;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       _usage
@@ -133,7 +134,7 @@ declare -A build_opts
 big_jobs=0
 build_jobs["ufs"]=8
 big_jobs=$((big_jobs+1))
-build_opts["ufs"]="${_wave_opt} ${_hydro_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug}"
+build_opts["ufs"]="${_wave_opt} ${_hydro_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug} ${_ufs_exec}"
 
 build_jobs["upp"]=1
 build_opts["upp"]="${_build_debug}"
diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh
index 6432962a5a..478588f92b 100755
--- a/sorc/build_ufs.sh
+++ b/sorc/build_ufs.sh
@@ -8,8 +8,9 @@ APP="S2SWA"
 CCPP_SUITES="FV3_GFS_v17_p8_ugwpv1,FV3_GFS_v17_coupled_p8_ugwpv1,FV3_global_nest_v1"  # TODO: does the g-w need to build with all these CCPP_SUITES?
 PDLIB="ON"
 HYDRO="OFF"
+EXEC_NAME="gfs_model.x"
 
-while getopts ":da:fj:vwy" option; do
+while getopts ":da:fj:e:vwy" option; do
   case "${option}" in
     d) BUILD_TYPE="Debug";;
     a) APP="${OPTARG}";;
@@ -18,6 +19,7 @@ while getopts ":da:fj:vwy" option; do
     v) export BUILD_VERBOSE="YES";;
     w) PDLIB="OFF";;
     y) HYDRO="ON";;
+    e) EXEC_NAME="${OPTARG}";;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       ;;
@@ -40,13 +42,19 @@ if [[ ${BUILD_TYPE:-"Release"} = "DEBUG" ]] ; then
 elif [[ "${FASTER:-OFF}" == ON ]] ; then
     MAKE_OPT+=" -DFASTER=ON"
 fi
-COMPILE_NR=0
+
+case "${EXEC_NAME}" in
+  "gfs_model.x") COMPILE_ID=1 ;;
+  "gefs_model.x") COMPILE_ID=2 ;;
+  "sfs_model.x") COMPILE_ID=3 ;;
+  *) COMPILE_ID=0 ;;
+esac
 CLEAN_BEFORE=YES
 CLEAN_AFTER=NO
 
-BUILD_JOBS=${BUILD_JOBS:-8} ./tests/compile.sh "${MACHINE_ID}" "${MAKE_OPT}" "${COMPILE_NR}" "intel" "${CLEAN_BEFORE}" "${CLEAN_AFTER}"
-mv "./tests/fv3_${COMPILE_NR}.exe" ./tests/ufs_model.x
-mv "./tests/modules.fv3_${COMPILE_NR}.lua" ./tests/modules.ufs_model.lua
-cp "./modulefiles/ufs_common.lua" ./tests/ufs_common.lua
+BUILD_JOBS=${BUILD_JOBS:-8} ./tests/compile.sh "${MACHINE_ID}" "${MAKE_OPT}" "${COMPILE_ID}" "intel" "${CLEAN_BEFORE}" "${CLEAN_AFTER}"
+mv "./tests/fv3_${COMPILE_ID}.exe" "./tests/${EXEC_NAME}"
+if [[ ! -f "./tests/modules.ufs_model.lua" ]]; then mv "./tests/modules.fv3_${COMPILE_ID}.lua" "./tests/modules.ufs_model.lua"; fi
+if [[ ! -f "./tests/ufs_common.lua" ]]; then cp "./modulefiles/ufs_common.lua" ./tests/ufs_common.lua; fi
 
 exit 0

From 98f26a876326efe4a907d730400563bf3504b831 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 13 Nov 2024 21:41:47 -0500
Subject: [PATCH 02/37] use SFS_POST as a proxy for SFS

---
 parm/config/gefs/config.fcst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst
index 21c666c520..2405723be5 100644
--- a/parm/config/gefs/config.fcst
+++ b/parm/config/gefs/config.fcst
@@ -63,7 +63,11 @@ export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_O
 
 export FORECASTSH="${SCRgfs}/exglobal_forecast.sh"
 #export FORECASTSH="${SCRgfs}/exglobal_forecast.py"  # Temp. while this is worked on
-export FCSTEXEC="gefs_model.x"
+if [[ "${SFS_POST:-}" == "YES" ]]; then
+  export FCSTEXEC="sfs_model.x"
+else
+  export FCSTEXEC="gefs_model.x"
+fi
 
 #######################################################################
 # Model configuration

From 041e7e1c5381586a032b414d977108ae8122ee40 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 13 Nov 2024 22:02:25 -0500
Subject: [PATCH 03/37] link/copy the appropriate model executable

---
 sorc/link_workflow.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index b35b7ff35a..129832b6d3 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -298,8 +298,12 @@ do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" .
 done
 
-[[ -s "ufs_model.x" ]] && rm -f ufs_model.x
-${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/ufs_model.x" .
+[[ -s "gfs_model.x" ]] && rm -f gfs_model.x
+[[ -s "gefs_model.x" ]] && rm -f gefs_model.x
+[[ -s "sfs_model.x" ]] && rm -f sfs_model.x
+if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" .; fi
+if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" .; fi
+if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" .; fi
 
 [[ -s "upp.x" ]] && rm -f upp.x
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/exec/upp.x" .

From e536a3a5ae95cd7db7837cdfa21b8b4bc369637a Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 4 Dec 2024 11:05:41 -0500
Subject: [PATCH 04/37] create gfs/gefs versions of ww3 pre/post execs

---
 sorc/build_ufs.sh        |   5 +-
 sorc/build_ww3prepost.sh | 127 ++++++++-----------
 sorc/link_workflow.sh    | 259 ++++++++++++++++++++-------------------
 3 files changed, 187 insertions(+), 204 deletions(-)

diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh
index 478588f92b..863ca30b65 100755
--- a/sorc/build_ufs.sh
+++ b/sorc/build_ufs.sh
@@ -8,7 +8,7 @@ APP="S2SWA"
 CCPP_SUITES="FV3_GFS_v17_p8_ugwpv1,FV3_GFS_v17_coupled_p8_ugwpv1,FV3_global_nest_v1"  # TODO: does the g-w need to build with all these CCPP_SUITES?
 PDLIB="ON"
 HYDRO="OFF"
-EXEC_NAME="gfs_model.x"
+EXEC_NAME="ufs_model.x"
 
 while getopts ":da:fj:e:vwy" option; do
   case "${option}" in
@@ -44,10 +44,11 @@ elif [[ "${FASTER:-OFF}" == ON ]] ; then
 fi
 
 case "${EXEC_NAME}" in
+  "ufs_model.x") COMPILE_ID=0 ;;
   "gfs_model.x") COMPILE_ID=1 ;;
   "gefs_model.x") COMPILE_ID=2 ;;
   "sfs_model.x") COMPILE_ID=3 ;;
-  *) COMPILE_ID=0 ;;
+  *) echo "Unsupported executable name: ${EXEC_NAME}"; exit 1 ;;
 esac
 CLEAN_BEFORE=YES
 CLEAN_AFTER=NO
diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index 67ee5e1dc2..b6db132156 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -1,131 +1,104 @@
 #! /usr/bin/env bash
 set -x
 
-script_dir=$(dirname "${BASH_SOURCE[0]}")
-cd "${script_dir}" || exit 1
+readonly _HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}")")/.." && pwd -P)
+cd "${_HOMEgfs}/sorc" || exit 1
 
 # Default settings
-APP="S2SWA"
 PDLIB="ON"
 
 while getopts ":j:a:dvw" option; do
   case "${option}" in
-    a) APP="${OPTARG}";;
-    d) BUILD_TYPE="Debug";;
-    j) BUILD_JOBS="${OPTARG}";;
-    v) export BUILD_VERBOSE="YES";;
-    w) PDLIB="OFF";;
-    :)
-      echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
-      usage
-      ;;
-    *)
-      echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}"
-      usage
-      ;;
+  d) BUILD_TYPE="Debug" ;;
+  j) BUILD_JOBS="${OPTARG}" ;;
+  v) export BUILD_VERBOSE="YES" ;;
+  w) PDLIB="OFF" ;;
+  :)
+    echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
+    usage
+    ;;
+  *)
+    echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}"
+    usage
+    ;;
   esac
 done
 
-# Determine which switch to use
-if [[ "${APP}" == "ATMW" ]]; then
-  ww3switch="model/esmf/switch"
-else
-  if [[ "${PDLIB}" == "ON" ]]; then
-    ww3switch="model/bin/switch_meshcap_pdlib"
-  else
-    ww3switch="model/bin/switch_meshcap"
-  fi
-fi
-
-# Check final exec folder exists
-if [[ ! -d "../exec" ]]; then
-  mkdir ../exec
-fi
-
-finalexecdir="$( pwd -P )/../exec"
-
-#Determine machine and load modules
+# Determine machine and load modules
 set +x
-source "${script_dir}/ufs_model.fd/tests/detect_machine.sh"
-source "${script_dir}/ufs_model.fd/tests/module-setup.sh"
-module use "${script_dir}/ufs_model.fd/modulefiles"
+source "${_HOMEgfs}/sorc/ufs_model.fd/tests/detect_machine.sh"
+source "${_HOMEgfs}/sorc/ufs_model.fd/tests/module-setup.sh"
+module use "${_HOMEgfs}/sorc/ufs_model.fd/modulefiles"
 module load "ufs_${MACHINE_ID}.intel"
 set -x
 
 #Set WW3 directory, switch, prep and post exes
-cd ufs_model.fd/WW3 || exit 1
-WW3_DIR=$( pwd -P )
+cd "${_HOMEgfs}/sorc/ufs_model.fd/WW3" || exit 1
+WW3_DIR=$(pwd -P)
 export WW3_DIR
 export SWITCHFILE="${WW3_DIR}/${ww3switch}"
 
-# Build exes for prep jobs and post jobs:
-prep_exes="ww3_grid ww3_prep ww3_prnc ww3_grid"
-post_exes="ww3_outp ww3_outf ww3_outp ww3_gint ww3_ounf ww3_ounp ww3_grib"
+# Determine which switch to use
+if [[ "${PDLIB}" == "ON" ]]; then
+  ww3switch="model/bin/switch_meshcap_pdlib"
+  path_build="${WW3_DIR}/build_pdlib"
+  path_install="${WW3_DIR}/install_pdlib"
+else
+  ww3switch="model/bin/switch_meshcap"
+  path_build="${WW3_DIR}/build"
+  path_install="install"
+fi
 
 #create build directory:
-path_build="${WW3_DIR}/build_SHRD"
 [[ -d "${path_build}" ]] && rm -rf "${path_build}"
 mkdir -p "${path_build}" || exit 1
 cd "${path_build}" || exit 1
-echo "Forcing a SHRD build"
 
 buildswitch="${path_build}/switch"
 
-cat "${SWITCHFILE}" > "${path_build}/tempswitch"
-
-sed -e "s/DIST/SHRD/g"\
-    -e "s/OMPG / /g"\
-    -e "s/OMPH / /g"\
-    -e "s/MPIT / /g"\
-    -e "s/MPI / /g"\
-    -e "s/B4B / /g"\
-    -e "s/PDLIB / /g"\
-    -e "s/SCOTCH / /g"\
-    -e "s/METIS / /g"\
-    -e "s/NOGRB/NCEP2/g"\
-       "${path_build}/tempswitch" > "${path_build}/switch"
+cat "${SWITCHFILE}" >"${path_build}/tempswitch"
+
+sed -e "s/DIST/SHRD/g" \
+  -e "s/OMPG / /g" \
+  -e "s/OMPH / /g" \
+  -e "s/MPIT / /g" \
+  -e "s/MPI / /g" \
+  -e "s/B4B / /g" \
+  -e "s/PDLIB / /g" \
+  -e "s/SCOTCH / /g" \
+  -e "s/METIS / /g" \
+  -e "s/NOGRB/NCEP2/g" \
+  "${path_build}/tempswitch" >"${path_build}/switch"
 rm "${path_build}/tempswitch"
 
 echo "Switch file is ${buildswitch} with switches:"
 cat "${buildswitch}"
 
 #define cmake build options
-MAKE_OPT="-DCMAKE_INSTALL_PREFIX=install"
+MAKE_OPT="-DCMAKE_INSTALL_PREFIX=${path_install}"
 [[ ${BUILD_TYPE:-"Release"} = "Debug" ]] && MAKE_OPT+=" -DCMAKE_BUILD_TYPE=Debug"
 
 #Build executables:
 # shellcheck disable=SC2086
 cmake "${WW3_DIR}" -DSWITCH="${buildswitch}" ${MAKE_OPT}
 rc=$?
-if (( rc != 0 )); then
+if ((rc != 0)); then
   echo "Fatal error in cmake."
   exit "${rc}"
 fi
+
 make -j "${BUILD_JOBS:-8}"
 rc=$?
-if (( rc != 0 )); then
+if ((rc != 0)); then
   echo "Fatal error in make."
   exit "${rc}"
 fi
+
 make install
-if (( rc != 0 )); then
+rc=$?
+if ((rc != 0)); then
   echo "Fatal error in make install."
   exit "${rc}"
 fi
 
-# Copy to top-level exe directory
-for prog in ${prep_exes} ${post_exes}; do
-  cp "${path_build}/install/bin/${prog}" "${finalexecdir}/"
-  rc=$?
-  if (( rc != 0 )); then
-    echo "FATAL: Unable to copy ${path_build}/${prog} to ${finalexecdir} (Error code ${rc})"
-    exit "${rc}"
-  fi
-done
-
-#clean-up build directory:
-echo "executables are in ${finalexecdir}"
-echo "cleaning up ${path_build}"
-rm -rf "${path_build}"
-
 exit 0
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 129832b6d3..4afa924d8a 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -2,11 +2,11 @@
 
 #--make symbolic links for EMC installation and hardcopies for NCO delivery
 
-HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd )"
+HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd)"
 TRACE=NO source "${HOMEgfs}/ush/preamble.sh"
 
 function usage() {
-  cat << EOF
+  cat <<EOF
 Builds all of the global-workflow components by calling the individual build
   scripts in sequence.
 
@@ -25,26 +25,27 @@ RUN_ENVIR="emc"
 OPTIND=1
 while getopts ":ho-:" option; do
   case "${option}" in
-    h) usage ;;
-    o)
-      echo "-o option received, configuring for NCO"
-      RUN_ENVIR="nco";;
-    -)
-      if [[ "${OPTARG}" == "nest" ]]; then
-        LINK_NEST=ON
-      fi
-      ;;
-    :)
-      echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
-      usage
-      ;;
-    *)
-      echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}"
-      usage
-      ;;
+  h) usage ;;
+  o)
+    echo "-o option received, configuring for NCO"
+    RUN_ENVIR="nco"
+    ;;
+  -)
+    if [[ "${OPTARG}" == "nest" ]]; then
+      LINK_NEST=ON
+    fi
+    ;;
+  :)
+    echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
+    usage
+    ;;
+  *)
+    echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}"
+    usage
+    ;;
   esac
 done
-shift $((OPTIND-1))
+shift $((OPTIND - 1))
 
 # LINK is always ln, LINK_OR_COPY can be ln or cp depending on RUN_ENVIR being emc or nco, respectively
 LINK="ln -fs"
@@ -55,7 +56,7 @@ else
 fi
 
 # shellcheck disable=SC1091
-COMPILER="intel" source "${HOMEgfs}/sorc/gfs_utils.fd/ush/detect_machine.sh"  # (sets MACHINE_ID)
+COMPILER="intel" source "${HOMEgfs}/sorc/gfs_utils.fd/ush/detect_machine.sh" # (sets MACHINE_ID)
 # shellcheck disable=
 machine=$(echo "${MACHINE_ID}" | cut -d. -f1)
 
@@ -69,18 +70,18 @@ ${LINK_OR_COPY} "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/ru
 #--model fix fields
 #------------------------------
 case "${machine}" in
-  "wcoss2")   FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;;
-  "hera")     FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;;
-  "orion")    FIX_DIR="/work/noaa/global/glopara/fix" ;;
-  "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;;
-  "jet")      FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;;
-  "s4")       FIX_DIR="/data/prod/glopara/fix" ;;
-  "gaea")     FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
-  "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;;
-  *)
-    echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR"
-    exit 1
-    ;;
+"wcoss2") FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;;
+"hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;;
+"orion") FIX_DIR="/work/noaa/global/glopara/fix" ;;
+"hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;;
+"jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;;
+"s4") FIX_DIR="/data/prod/glopara/fix" ;;
+"gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
+"noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;;
+*)
+  echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR"
+  exit 1
+  ;;
 esac
 
 # Source fix version file
@@ -89,9 +90,9 @@ source "${HOMEgfs}/versions/fix.ver"
 # Link GDASapp python packages in ush/python
 packages=("jcb")
 for package in "${packages[@]}"; do
-    cd "${HOMEgfs}/ush/python" || exit 1
-    [[ -s "${package}" ]] && rm -f "${package}"
-    ${LINK} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" .
+  cd "${HOMEgfs}/ush/python" || exit 1
+  [[ -s "${package}" ]] && rm -f "${package}"
+  ${LINK} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" .
 done
 
 # Link fix directories
@@ -100,20 +101,19 @@ if [[ -n "${FIX_DIR}" ]]; then
 fi
 cd "${HOMEgfs}/fix" || exit 1
 for dir in aer \
-            am \
-            chem \
-            cice \
-            cpl \
-            datm \
-            gsi \
-            lut \
-            mom6 \
-            orog \
-            sfc_climo \
-            ugwd \
-            verif \
-            wave
-do
+  am \
+  chem \
+  cice \
+  cpl \
+  datm \
+  gsi \
+  lut \
+  mom6 \
+  orog \
+  sfc_climo \
+  ugwd \
+  verif \
+  wave; do
   if [[ -d "${dir}" ]]; then
     [[ "${RUN_ENVIR}" == "nco" ]] && chmod -R 755 "${dir}"
     rm -rf "${dir}"
@@ -122,10 +122,9 @@ do
   ${LINK_OR_COPY} "${FIX_DIR}/${dir}/${!fix_ver}" "${dir}"
 done
 # global-nest uses different versions of orog and ugwd
-if [[ "${LINK_NEST:-OFF}" == "ON" ]] ; then
+if [[ "${LINK_NEST:-OFF}" == "ON" ]]; then
   for dir in orog \
-             ugwd
-  do
+    ugwd; do
     nestdir=${dir}_nest
     if [[ -d "${nestdir}" ]]; then
       [[ "${RUN_ENVIR}" == "nco" ]] && chmod -R 755 "${nestdir}"
@@ -144,26 +143,23 @@ cd "${HOMEgfs}/parm/ufs" || exit 1
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/noahmptable.tbl" .
 
 cd "${HOMEgfs}/parm/post" || exit 1
-for file in params_grib2_tbl_new nam_micro_lookup.dat
-do
+for file in params_grib2_tbl_new nam_micro_lookup.dat; do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" .
 done
-for dir in gfs gefs sfs
-do
+for dir in gfs gefs sfs; do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
-for file in ice.csv ocean.csv ocnicepost.nml.jinja2
-do
+for file in ice.csv ocean.csv ocnicepost.nml.jinja2; do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/parm/ocnicepost/${file}" .
 done
 
 cd "${HOMEgfs}/scripts" || exit 8
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/scripts/exemcsfc_global_sfc_prep.sh" .
 if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
-  declare -a gdas_scripts=(exglobal_prep_ocean_obs.py \
-                           exgdas_global_marine_analysis_ecen.py \
-                           )
-  for gdas_script in "${gdas_scripts[@]}" ; do
+  declare -a gdas_scripts=(exglobal_prep_ocean_obs.py
+    exgdas_global_marine_analysis_ecen.py
+  )
+  for gdas_script in "${gdas_scripts[@]}"; do
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/scripts/${gdas_script}" .
   done
 fi
@@ -171,31 +167,31 @@ cd "${HOMEgfs}/ush" || exit 8
 for file in emcsfc_ice_blend.sh global_cycle_driver.sh emcsfc_snow.sh global_cycle.sh; do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/ush/${file}" .
 done
-for file in make_ntc_bull.pl make_NTC_file.pl make_tif.sh month_name.sh ; do
+for file in make_ntc_bull.pl make_NTC_file.pl make_tif.sh month_name.sh; do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/ush/${file}" .
 done
 
 # Link these templates from ufs-weather-model
 cd "${HOMEgfs}/parm/ufs" || exit 1
-declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"\
-                          "MOM_input_025.IN" "MOM_input_050.IN" "MOM_input_100.IN" "MOM_input_500.IN" \
-                          "MOM6_data_table.IN" \
-                          "ice_in.IN" \
-                          "ufs.configure.atm.IN" \
-                          "ufs.configure.atm_esmf.IN" \
-                          "ufs.configure.atmaero.IN" \
-                          "ufs.configure.atmaero_esmf.IN" \
-                          "ufs.configure.s2s.IN" \
-                          "ufs.configure.s2s_esmf.IN" \
-                          "ufs.configure.s2sa.IN" \
-                          "ufs.configure.s2sa_esmf.IN" \
-                          "ufs.configure.s2sw.IN" \
-                          "ufs.configure.s2sw_esmf.IN" \
-                          "ufs.configure.s2swa.IN" \
-                          "ufs.configure.s2swa_esmf.IN" \
-                          "ufs.configure.leapfrog_atm_wav.IN" \
-                          "ufs.configure.leapfrog_atm_wav_esmf.IN" \
-                          "post_itag_gfs")
+declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"
+  "MOM_input_025.IN" "MOM_input_050.IN" "MOM_input_100.IN" "MOM_input_500.IN"
+  "MOM6_data_table.IN"
+  "ice_in.IN"
+  "ufs.configure.atm.IN"
+  "ufs.configure.atm_esmf.IN"
+  "ufs.configure.atmaero.IN"
+  "ufs.configure.atmaero_esmf.IN"
+  "ufs.configure.s2s.IN"
+  "ufs.configure.s2s_esmf.IN"
+  "ufs.configure.s2sa.IN"
+  "ufs.configure.s2sa_esmf.IN"
+  "ufs.configure.s2sw.IN"
+  "ufs.configure.s2sw_esmf.IN"
+  "ufs.configure.s2swa.IN"
+  "ufs.configure.s2swa_esmf.IN"
+  "ufs.configure.leapfrog_atm_wav.IN"
+  "ufs.configure.leapfrog_atm_wav_esmf.IN"
+  "post_itag_gfs")
 for file in "${ufs_templates[@]}"; do
   [[ -s "${file}" ]] && rm -f "${file}"
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" .
@@ -206,7 +202,6 @@ cd "${HOMEgfs}/ush" || exit 1
 [[ -s "atparse.bash" ]] && rm -f "atparse.bash"
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/atparse.bash" .
 
-
 #------------------------------
 #--add GDASApp fix directory
 #------------------------------
@@ -216,7 +211,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
   cd gdas || exit 1
   for gdas_sub in fv3jedi gsibec obs soca aero; do
     if [[ -d "${gdas_sub}" ]]; then
-       rm -rf "${gdas_sub}"
+      rm -rf "${gdas_sub}"
     fi
     fix_ver="gdas_${gdas_sub}_ver"
     ${LINK_OR_COPY} "${FIX_DIR}/gdas/${gdas_sub}/${!fix_ver}" "${gdas_sub}"
@@ -240,13 +235,13 @@ fi
 #------------------------------
 if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
   cd "${HOMEgfs}/ush" || exit 1
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/soca"                              .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ufsda"                              .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/jediinc2fv3.py"                     .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py"    .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py"    .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py"    .
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py"           .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/soca" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ufsda" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/jediinc2fv3.py" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py" .
   declare -a gdasapp_ocn_insitu_profile_platforms=("argo" "bathy" "glider" "marinemammal" "tesac" "xbtctd")
   for platform in "${gdasapp_ocn_insitu_profile_platforms[@]}"; do
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/marine/bufr2ioda_insitu_profile_${platform}.py" .
@@ -286,14 +281,13 @@ fi
 #--link executables
 #------------------------------
 
-if [[ ! -d "${HOMEgfs}/exec" ]]; then mkdir "${HOMEgfs}/exec" || exit 1 ; fi
+if [[ ! -d "${HOMEgfs}/exec" ]]; then mkdir "${HOMEgfs}/exec" || exit 1; fi
 cd "${HOMEgfs}/exec" || exit 1
 
 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \
   syndat_maksynrc.x syndat_qctropcy.x tocsbufr.x overgridid.x rdbfmsua.x \
   mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x \
-  ensadd.x ensppf.x ensstat.x wave_stat.x
-do
+  ensadd.x ensppf.x ensstat.x wave_stat.x; do
   [[ -s "${utilexe}" ]] && rm -f "${utilexe}"
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" .
 done
@@ -305,12 +299,30 @@ if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" ]]; then ${LINK_OR_COP
 if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" .; fi
 if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" .; fi
 
+# WW3 pre/post executables
+declare -a ww3_exes=("ww3_grid" "ww3_prep" "ww3_prnc" "ww3_outp" "ww3_outf" "ww3_gint" "ww3_ounf" "ww3_ounp" "ww3_grib")
+if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install" ]]; then # installed with PDLIB=OFF (Used in GEFS)
+  for ww3exe in "${ww3_exes[@]}"; do
+    [[ -s "gefs_${ww3exe}" ]] && rm -f "gefs_${HOMEgfs}/exec/${ww3exe}"
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/bin/${ww3exe}" \
+      "${HOMEgfs}/exec/gefs_${ww3exe}"
+  done
+fi
+
+if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install_pdlib" ]]; then # installed with PDLIB=ON (Used in GFS, may be SFS later)
+  for ww3exe in "${ww3_exes[@]}"; do
+    [[ -s "gfs_${ww3exe}" ]] && rm -f "gfs_${ww3exe}"
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install_pdlib/bin/${ww3exe}" \
+      "${HOMEgfs}/exec/gfs_${ww3exe}"
+  done
+fi
+
 [[ -s "upp.x" ]] && rm -f upp.x
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/exec/upp.x" .
 
 for ufs_utilsexe in emcsfc_ice_blend emcsfc_snow2mdl global_cycle fregrid; do
-    [[ -s "${ufs_utilsexe}" ]] && rm -f "${ufs_utilsexe}"
-    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/exec/${ufs_utilsexe}" .
+  [[ -s "${ufs_utilsexe}" ]] && rm -f "${ufs_utilsexe}"
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/exec/${ufs_utilsexe}" .
 done
 
 # GSI
@@ -325,8 +337,7 @@ fi
 if [[ -d "${HOMEgfs}/sorc/gsi_utils.fd/install" ]]; then
   for exe in calc_analysis.x calc_increment_ens_ncio.x calc_increment_ens.x \
     getsfcensmeanp.x getsigensmeanp_smooth.x getsigensstatp.x \
-    interp_inc.x recentersigp.x
-  do
+    interp_inc.x recentersigp.x; do
     [[ -s "${exe}" ]] && rm -f "${exe}"
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/gsi_utils.fd/install/bin/${exe}" .
   done
@@ -335,8 +346,7 @@ fi
 # GSI Monitor
 if [[ -d "${HOMEgfs}/sorc/gsi_monitor.fd/install" ]]; then
   for exe in oznmon_horiz.x oznmon_time.x radmon_angle.x \
-    radmon_bcoef.x radmon_bcor.x radmon_time.x
-  do
+    radmon_bcoef.x radmon_bcor.x radmon_time.x; do
     [[ -s "${exe}" ]] && rm -f "${exe}"
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/gsi_monitor.fd/install/bin/${exe}" .
   done
@@ -344,24 +354,24 @@ fi
 
 # GDASApp
 if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
-  declare -a JEDI_EXE=("gdas.x" \
-                       "gdas_soca_gridgen.x" \
-                       "gdas_soca_error_covariance_toolbox.x" \
-                       "gdas_fv3jedi_error_covariance_toolbox.x" \
-                       "gdas_soca_setcorscales.x" \
-                       "gdas_soca_diagb.x" \
-                       "fv3jedi_plot_field.x" \
-                       "gdasapp_chem_diagb.x" \
-                       "fv3jedi_fv3inc.x" \
-                       "gdas_ens_handler.x" \
-                       "gdas_incr_handler.x" \
-                       "gdas_obsprovider2ioda.x" \
-                       "gdas_socahybridweights.x" \
-                       "gdassoca_obsstats.x" \
-                       "gdasapp_land_ensrecenter.x" \
-                       "bufr2ioda.x" \
-                       "calcfIMS.exe" \
-                       "apply_incr.exe" )
+  declare -a JEDI_EXE=("gdas.x"
+    "gdas_soca_gridgen.x"
+    "gdas_soca_error_covariance_toolbox.x"
+    "gdas_fv3jedi_error_covariance_toolbox.x"
+    "gdas_soca_setcorscales.x"
+    "gdas_soca_diagb.x"
+    "fv3jedi_plot_field.x"
+    "gdasapp_chem_diagb.x"
+    "fv3jedi_fv3inc.x"
+    "gdas_ens_handler.x"
+    "gdas_incr_handler.x"
+    "gdas_obsprovider2ioda.x"
+    "gdas_socahybridweights.x"
+    "gdassoca_obsstats.x"
+    "gdasapp_land_ensrecenter.x"
+    "bufr2ioda.x"
+    "calcfIMS.exe"
+    "apply_incr.exe")
   for gdasexe in "${JEDI_EXE[@]}"; do
     [[ -s "${gdasexe}" ]] && rm -f "${gdasexe}"
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/${gdasexe}" .
@@ -411,7 +421,7 @@ if [[ -d gsi_utils.fd ]]; then
   ${LINK} gsi_utils.fd/src/netcdf_io/interp_inc.fd .
 fi
 
-if [[ -d gsi_monitor.fd ]] ; then
+if [[ -d gsi_monitor.fd ]]; then
   [[ -d oznmon_horiz.fd ]] && rm -rf oznmon_horiz.fd
   ${LINK} gsi_monitor.fd/src/Ozone_Monitor/nwprod/oznmon_shared/sorc/oznmon_horiz.fd .
 
@@ -431,9 +441,9 @@ if [[ -d gsi_monitor.fd ]] ; then
   ${LINK} gsi_monitor.fd/src/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd radmon_time.fd
 fi
 
-for prog in global_cycle.fd emcsfc_ice_blend.fd emcsfc_snow2mdl.fd ;do
-    [[ -d "${prog}" ]] && rm -rf "${prog}"
-    ${LINK} "ufs_utils.fd/sorc/${prog}" "${prog}"
+for prog in global_cycle.fd emcsfc_ice_blend.fd emcsfc_snow2mdl.fd; do
+  [[ -d "${prog}" ]] && rm -rf "${prog}"
+  ${LINK} "ufs_utils.fd/sorc/${prog}" "${prog}"
 done
 
 for prog in enkf_chgres_recenter_nc.fd \
@@ -451,8 +461,7 @@ for prog in enkf_chgres_recenter_nc.fd \
   tocsbufr.fd \
   vint.fd \
   webtitle.fd \
-  ocnicepost.fd
-do
+  ocnicepost.fd; do
   if [[ -d "${prog}" ]]; then rm -rf "${prog}"; fi
   ${LINK_OR_COPY} "gfs_utils.fd/src/${prog}" .
 done

From b8eb15d1ec942d51dd6b234acb7778e1777ab7d8 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Dec 2024 10:48:30 -0500
Subject: [PATCH 05/37] use NET specific executable names

---
 scripts/exgfs_wave_post_pnt.sh | 10 +++++-----
 sorc/link_workflow.sh          |  5 +++--
 ush/wave_grib2_sbs.sh          | 19 ++++++++++---------
 ush/wave_grid_interp_sbs.sh    | 13 +++++++------
 ush/wave_grid_moddef.sh        | 20 ++++++++++----------
 ush/wave_outp_spec.sh          | 12 ++++++------
 ush/wave_prnc_cur.sh           | 10 +++++-----
 ush/wave_prnc_ice.sh           | 20 ++++++++++----------
 8 files changed, 56 insertions(+), 53 deletions(-)

diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh
index 06769303a1..f2dfdf9399 100755
--- a/scripts/exgfs_wave_post_pnt.sh
+++ b/scripts/exgfs_wave_post_pnt.sh
@@ -271,18 +271,18 @@ source "${USHgfs}/preamble.sh"
     rm -f buoy_tmp.loc buoy_log.ww3 ww3_oup.inp
     ${NLN} ./out_pnt.${waveuoutpGRD} ./out_pnt.ww3
     ${NLN} ./mod_def.${waveuoutpGRD} ./mod_def.ww3
-    export pgm=ww3_outp;. prep_step
-    ${EXECgfs}/ww3_outp > buoy_lst.loc 2>&1
-    export err=$?;err_chk
 
+    export pgm="${NET,,}_ww3_outp.x"
+    . prep_step
 
+    "${EXECgfs}/${pgm}" > buoy_lst.loc 2>&1
+    export err=$?;err_chk
     if [ "$err" != '0' ] && [ ! -f buoy_log.ww3 ]
     then
-      pgm=wave_post
       set +x
       echo ' '
       echo '******************************************** '
-      echo '*** FATAL ERROR : ERROR IN ww3_outp *** '
+      echo '*** FATAL ERROR : ERROR IN ${pgm} *** '
       echo '******************************************** '
       echo ' '
       cat buoy_tmp.loc
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 4afa924d8a..ead835eed8 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -301,11 +301,12 @@ if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" ]]; then ${LINK_OR_COP
 
 # WW3 pre/post executables
 declare -a ww3_exes=("ww3_grid" "ww3_prep" "ww3_prnc" "ww3_outp" "ww3_outf" "ww3_gint" "ww3_ounf" "ww3_ounp" "ww3_grib")
+# TODO: ww3_prep, ww3_outf, ww3_ounf, ww3_ounp are not used in the workflow # FIXME or remove them from the list
 if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install" ]]; then # installed with PDLIB=OFF (Used in GEFS)
   for ww3exe in "${ww3_exes[@]}"; do
     [[ -s "gefs_${ww3exe}" ]] && rm -f "gefs_${HOMEgfs}/exec/${ww3exe}"
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/bin/${ww3exe}" \
-      "${HOMEgfs}/exec/gefs_${ww3exe}"
+      "${HOMEgfs}/exec/gefs_${ww3exe}.x"
   done
 fi
 
@@ -313,7 +314,7 @@ if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install_pdlib" ]]; then # installed w
   for ww3exe in "${ww3_exes[@]}"; do
     [[ -s "gfs_${ww3exe}" ]] && rm -f "gfs_${ww3exe}"
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install_pdlib/bin/${ww3exe}" \
-      "${HOMEgfs}/exec/gfs_${ww3exe}"
+      "${HOMEgfs}/exec/gfs_${ww3exe}.x"
   done
 fi
 
diff --git a/ush/wave_grib2_sbs.sh b/ush/wave_grib2_sbs.sh
index 99f89f3f37..1a7b3179b6 100755
--- a/ush/wave_grib2_sbs.sh
+++ b/ush/wave_grib2_sbs.sh
@@ -136,27 +136,28 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
 
   # 1.b Run GRIB packing program
 
+
+  export pgm="${NET,,}_ww3_grib.x"
+  . prep_step
+
   set +x
-  echo "   Run ww3_grib2"
-  echo "   Executing ${EXECgfs}/ww3_grib"
+  echo "   Executing ${EXECgfs}/${pgm}"
   set_trace
 
-  export pgm=ww3_grib;. prep_step
-  "${EXECgfs}/ww3_grib" > "grib2_${grdnam}_${FH3}.out" 2>&1
+  "${EXECgfs}/${pgm}" > "grib2_${grdnam}_${FH3}.out" 2>&1
   export err=$?;err_chk
-
   if [ ! -s gribfile ]; then
     set +x
     echo ' '
     echo '************************************************ '
-    echo '*** FATAL ERROR : ERROR IN ww3_grib encoding *** '
+    echo '*** FATAL ERROR : ERROR IN ${pgm} encoding *** '
     echo '************************************************ '
     echo ' '
     set_trace
     exit 3
   fi
 
-  if (( fhr > 0 )); then 
+  if (( fhr > 0 )); then
     ${WGRIB2} gribfile -set_date "${PDY}${cyc}" -set_ftime "${fhr} hour fcst" -grib "${COMOUT_WAVE_GRID}/${outfile}"
     err=$?
   else
@@ -169,7 +170,7 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
     set +x
     echo ' '
     echo '********************************************* '
-    echo '*** FATAL ERROR : ERROR IN ww3_grib2 *** '
+    echo '*** FATAL ERROR : ERROR IN ${pgm} *** '  # FIXME: This is not an error in $pgm, but in WGRIB2
     echo '********************************************* '
     echo ' '
     set_trace
@@ -197,7 +198,7 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
     set +x
     echo ' '
     echo '********************************************* '
-    echo '*** FATAL ERROR : ERROR IN ww3_grib2 *** '
+    echo '*** FATAL ERROR : ERROR IN ${pgm} *** '
     echo '********************************************* '
     echo ' '
     echo " Error in moving grib file ${outfile} to com"
diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh
index 31b7808c16..e848badbe8 100755
--- a/ush/wave_grid_interp_sbs.sh
+++ b/ush/wave_grid_interp_sbs.sh
@@ -133,17 +133,18 @@ source "${USHgfs}/preamble.sh"
 
 # 1.b Run interpolation code
 
+  export pgm="${NET,,}_ww3_gint.x"
+  . prep_step
+
   set +x
-  echo "   Run ww3_gint
-  echo "   Executing ${EXECgfs}/ww3_gint
+  echo "   Executing ${pgm}"
   set_trace
 
-  export pgm=ww3_gint;. prep_step
-  ${EXECgfs}/ww3_gint 1> gint.${grdID}.out 2>&1
+  "${EXECgfs}/${pgm}" 1> gint.${grdID}.out 2>&1
   export err=$?;err_chk
 
 # Write interpolation file to main TEMP dir area if not there yet
-  if [ "wht_OK" = 'no' ]
+  if [ "wht_OK" = 'no' ]  # FIXME: This is never going to evaluate to true, wht_OK is a string and needs to be ${wht_OK}
   then
     cp -f ./WHTGRIDINT.bin ${DATA}/ww3_gint.WHTGRIDINT.bin.${grdID}
     cp -f ./WHTGRIDINT.bin ${FIXgfs}/wave/ww3_gint.WHTGRIDINT.bin.${grdID}
@@ -155,7 +156,7 @@ source "${USHgfs}/preamble.sh"
     set +x
     echo ' '
     echo '*************************************************** '
-    echo '*** FATAL ERROR : ERROR IN ww3_gint interpolation * '
+    echo '*** FATAL ERROR : ERROR IN ${pgm} interpolation * '
     echo '*************************************************** '
     echo ' '
     set_trace
diff --git a/ush/wave_grid_moddef.sh b/ush/wave_grid_moddef.sh
index 1e8c44054a..bd2a9a3deb 100755
--- a/ush/wave_grid_moddef.sh
+++ b/ush/wave_grid_moddef.sh
@@ -1,5 +1,5 @@
 #! /usr/bin/env bash
-                                                                   
+
 ################################################################################
 #
 # UNIX Script Documentation Block
@@ -72,27 +72,27 @@ source "${USHgfs}/preamble.sh"
   fi
 
 # --------------------------------------------------------------------------- #
-# 2.  Create mod_def file 
+# 2.  Create mod_def file
 
   set +x
   echo ' '
   echo '   Creating mod_def file ...'
-  echo "   Executing ${EXECgfs}/ww3_grid"
+  echo "   Executing ${EXECgfs}/${NET,,}_ww3_grid.x"
   echo ' '
   set_trace
- 
-  rm -f ww3_grid.inp 
+
+  rm -f ww3_grid.inp
   ${NLN} ../ww3_grid.inp.$grdID ww3_grid.inp
 
   if [ -f ../${grdID}.msh ]
   then
-     rm -f ${grdID}.msh 
+     rm -f ${grdID}.msh
      ${NLN} ../${grdID}.msh ${grdID}.msh
   fi
 
 
- 
-  "${EXECgfs}/ww3_grid" 1> "grid_${grdID}.out" 2>&1
+
+  "${EXECgfs}/${NET,,}_ww3_grid.x" 1> "grid_${grdID}.out" 2>&1
   err=$?
 
   if [ "$err" != '0' ]
@@ -100,13 +100,13 @@ source "${USHgfs}/preamble.sh"
     set +x
     echo ' '
     echo '******************************************** '
-    echo '*** FATAL ERROR : ERROR IN ww3_grid *** '
+    echo '*** FATAL ERROR : ERROR IN ${NET,,}_ww3_grid.x *** '
     echo '******************************************** '
     echo ' '
     set_trace
     exit 3
   fi
- 
+
   if [[ -f mod_def.ww3 ]]
   then
     cp mod_def.ww3 "${COMOUT_WAVE_PREP}/${RUN}wave.mod_def.${grdID}"
diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh
index 37accbae49..ae818171ca 100755
--- a/ush/wave_outp_spec.sh
+++ b/ush/wave_outp_spec.sh
@@ -161,21 +161,21 @@ source "${USHgfs}/preamble.sh"
 
 # 2.b Run the postprocessor
 
+  export pgm="${NET,,}_ww3_outp.x"
+  . prep_step
+
   set +x
-  echo "   Executing ${EXECgfs}/ww3_outp"
+  echo "   Executing ${EXECgfs}/${pgm}"
   set_trace
 
-  export pgm=ww3_outp;. prep_step
-  ${EXECgfs}/ww3_outp 1> outp_${specdir}_${buoy}.out 2>&1
+  "${EXECgfs}/${pgm}" 1> outp_${specdir}_${buoy}.out 2>&1
   export err=$?;err_chk
-
-
   if [ "$err" != '0' ]
   then
     set +x
     echo ' '
     echo '******************************************** '
-    echo '*** FATAL ERROR : ERROR IN ww3_outp *** '
+    echo '*** FATAL ERROR : ERROR IN ${pgm} *** '
     echo '******************************************** '
     echo ' '
     set_trace
diff --git a/ush/wave_prnc_cur.sh b/ush/wave_prnc_cur.sh
index 927710c581..02064427f4 100755
--- a/ush/wave_prnc_cur.sh
+++ b/ush/wave_prnc_cur.sh
@@ -74,22 +74,22 @@ rm -f cur.nc
 ${NLN} "cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc" "cur.nc"
 ${NLN} "${DATA}/mod_def.${WAVECUR_FID}" ./mod_def.ww3
 
-export pgm=ww3_prnc;. prep_step
-${EXECgfs}/ww3_prnc 1> prnc_${WAVECUR_FID}_${ymdh_rtofs}.out 2>&1
+export pgm="${NET,,}_ww3_prnc.x"
+. prep_step
 
+"${EXECgfs}/${pgm}" 1> prnc_${WAVECUR_FID}_${ymdh_rtofs}.out 2>&1
 export err=$?; err_chk
-
 if [ "$err" != '0' ]
 then
   cat prnc_${WAVECUR_FID}_${ymdh_rtofs}.out
   set $setoff
   echo ' '
   echo '******************************************** '
-  echo '*** WARNING: NON-FATAL ERROR IN ww3_prnc *** '
+  echo '*** WARNING: NON-FATAL ERROR IN ${pgm} *** '
   echo '******************************************** '
   echo ' '
   set $seton
-  echo "WARNING: NON-FATAL ERROR IN ww3_prnc."
+  echo "WARNING: NON-FATAL ERROR IN ${pgm}."
   exit 4
 fi
 
diff --git a/ush/wave_prnc_ice.sh b/ush/wave_prnc_ice.sh
index be089c30bd..890df71973 100755
--- a/ush/wave_prnc_ice.sh
+++ b/ush/wave_prnc_ice.sh
@@ -32,7 +32,7 @@ source "${USHgfs}/preamble.sh"
 # 0.a Basic modes of operation
 
   cd $DATA
-  
+
   rm -rf ice
   mkdir ice
   cd ice
@@ -57,7 +57,7 @@ source "${USHgfs}/preamble.sh"
   if [[ -z "${YMDH}" ]] || [[ -z "${cycle}" ]] || \
      [[ -z "${COMOUT_WAVE_PREP}" ]] || [[ -z "${FIXgfs}" ]] || [[ -z "${EXECgfs}" ]] || \
      [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${WAVEICE_FID}" ]] || [[ -z "${COMIN_OBS}" ]]; then
-  
+
     set +x
     echo ' '
     echo '**************************************************'
@@ -129,7 +129,7 @@ source "${USHgfs}/preamble.sh"
   fi
 
   rm -f wgrib.out
-  rm -f ice.grib 
+  rm -f ice.grib
   rm -f ice.index
 
 
@@ -142,14 +142,14 @@ source "${USHgfs}/preamble.sh"
 
   cp -f ${DATA}/ww3_prnc.ice.$WAVEICE_FID.inp.tmpl ww3_prnc.inp
 
-  export pgm=ww3_prnc;. prep_step
+  export pgm="${NET,,}_ww3_prnc.x"
+  . prep_step
 
-  ${EXECgfs}/ww3_prnc 1> prnc_${WAVEICE_FID}_${cycle}.out 2>&1
+  "${EXECgfs}/${pgm}" 1> prnc_${WAVEICE_FID}_${cycle}.out 2>&1
   export err=$?; err_chk
-
   if [ "$err" != '0' ]
   then
-    cat prnc_${WAVEICE_FID}_${cycle}.out 
+    cat prnc_${WAVEICE_FID}_${cycle}.out
     set +x
     echo ' '
     echo '******************************************** '
@@ -170,13 +170,13 @@ source "${USHgfs}/preamble.sh"
 # and only WAV_MOD_ID if WW3ATMIENS=F
 #
   if [ "${WW3ATMIENS}" = "T" ]
-  then 
+  then
     icefile=${WAV_MOD_TAG}.${WAVEICE_FID}.$cycle.ice
   elif [ "${WW3ATMIENS}" = "F" ]
-  then 
+  then
     icefile=${RUN}wave.${WAVEICE_FID}.$cycle.ice
   fi
- 
+
   set +x
   echo "   Saving ice.ww3 as ${COMOUT_WAVE_PREP}/${icefile}"
   set_trace

From f0bf57c445ff4d81971e32b846dae31393ff7ea0 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Dec 2024 10:57:54 -0500
Subject: [PATCH 06/37] fix errors detected by shellnorms

---
 ush/wave_grib2_sbs.sh       | 6 +++---
 ush/wave_grid_interp_sbs.sh | 2 +-
 ush/wave_grid_moddef.sh     | 5 +++--
 ush/wave_prnc_cur.sh        | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/ush/wave_grib2_sbs.sh b/ush/wave_grib2_sbs.sh
index 1a7b3179b6..6db0f50e4b 100755
--- a/ush/wave_grib2_sbs.sh
+++ b/ush/wave_grib2_sbs.sh
@@ -170,7 +170,7 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
     set +x
     echo ' '
     echo '********************************************* '
-    echo '*** FATAL ERROR : ERROR IN ${pgm} *** '  # FIXME: This is not an error in $pgm, but in WGRIB2
+    echo "*** FATAL ERROR : ERROR IN ${pgm} *** "  # FIXME: This is not an error in $pgm, but in WGRIB2
     echo '********************************************* '
     echo ' '
     set_trace
@@ -198,7 +198,7 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
     set +x
     echo ' '
     echo '********************************************* '
-    echo '*** FATAL ERROR : ERROR IN ${pgm} *** '
+    echo "*** FATAL ERROR : ERROR IN ${pgm} *** "
     echo '********************************************* '
     echo ' '
     echo " Error in moving grib file ${outfile} to com"
@@ -210,7 +210,7 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
     set +x
     echo ' '
     echo '*************************************************** '
-    echo '*** FATAL ERROR : ERROR IN ww3_grib2 INDEX FILE *** '
+    echo "*** FATAL ERROR : ERROR IN ${pgm} INDEX FILE *** "
     echo '*************************************************** '
     echo ' '
     echo " Error in moving grib file ${outfile}.idx to com"
diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh
index e848badbe8..190dff221b 100755
--- a/ush/wave_grid_interp_sbs.sh
+++ b/ush/wave_grid_interp_sbs.sh
@@ -156,7 +156,7 @@ source "${USHgfs}/preamble.sh"
     set +x
     echo ' '
     echo '*************************************************** '
-    echo '*** FATAL ERROR : ERROR IN ${pgm} interpolation * '
+    echo "*** FATAL ERROR : ERROR IN ${pgm} interpolation * "
     echo '*************************************************** '
     echo ' '
     set_trace
diff --git a/ush/wave_grid_moddef.sh b/ush/wave_grid_moddef.sh
index bd2a9a3deb..dd0ee204f6 100755
--- a/ush/wave_grid_moddef.sh
+++ b/ush/wave_grid_moddef.sh
@@ -91,8 +91,9 @@ source "${USHgfs}/preamble.sh"
   fi
 
 
+  export pgm="${NET,,}_ww3_grid.x"
 
-  "${EXECgfs}/${NET,,}_ww3_grid.x" 1> "grid_${grdID}.out" 2>&1
+  "${EXECgfs}/${pgm}" 1> "grid_${grdID}.out" 2>&1
   err=$?
 
   if [ "$err" != '0' ]
@@ -100,7 +101,7 @@ source "${USHgfs}/preamble.sh"
     set +x
     echo ' '
     echo '******************************************** '
-    echo '*** FATAL ERROR : ERROR IN ${NET,,}_ww3_grid.x *** '
+    echo "*** FATAL ERROR : ERROR IN ${pgm} *** "
     echo '******************************************** '
     echo ' '
     set_trace
diff --git a/ush/wave_prnc_cur.sh b/ush/wave_prnc_cur.sh
index 02064427f4..b084b8dcac 100755
--- a/ush/wave_prnc_cur.sh
+++ b/ush/wave_prnc_cur.sh
@@ -85,7 +85,7 @@ then
   set $setoff
   echo ' '
   echo '******************************************** '
-  echo '*** WARNING: NON-FATAL ERROR IN ${pgm} *** '
+  echo "*** WARNING: NON-FATAL ERROR IN ${pgm} *** "
   echo '******************************************** '
   echo ' '
   set $seton

From 3bd366a790f8cc4e3196a6e8912591270c6916bd Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Dec 2024 11:00:17 -0500
Subject: [PATCH 07/37] one shellnorm error escaped

---
 ush/wave_grib2_sbs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_grib2_sbs.sh b/ush/wave_grib2_sbs.sh
index 6db0f50e4b..22eb361a0d 100755
--- a/ush/wave_grib2_sbs.sh
+++ b/ush/wave_grib2_sbs.sh
@@ -150,7 +150,7 @@ if [[ ! -s "${COMOUT_WAVE_GRID}/${outfile}.idx" ]]; then
     set +x
     echo ' '
     echo '************************************************ '
-    echo '*** FATAL ERROR : ERROR IN ${pgm} encoding *** '
+    echo "*** FATAL ERROR : ERROR IN ${pgm} encoding *** "
     echo '************************************************ '
     echo ' '
     set_trace

From b102c079998db81099086febec62b456bb5cf184 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Dec 2024 11:01:44 -0500
Subject: [PATCH 08/37] one shellnorm error escaped

---
 scripts/exgfs_wave_post_pnt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh
index f2dfdf9399..9b40852092 100755
--- a/scripts/exgfs_wave_post_pnt.sh
+++ b/scripts/exgfs_wave_post_pnt.sh
@@ -282,7 +282,7 @@ source "${USHgfs}/preamble.sh"
       set +x
       echo ' '
       echo '******************************************** '
-      echo '*** FATAL ERROR : ERROR IN ${pgm} *** '
+      echo "*** FATAL ERROR : ERROR IN ${pgm} *** "
       echo '******************************************** '
       echo ' '
       cat buoy_tmp.loc

From 8bee97c07a3f667f38d35faa436cdc6df6282991 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Dec 2024 16:03:02 -0500
Subject: [PATCH 09/37] minor fixes for typos and such

---
 sorc/build_ww3prepost.sh | 12 ++++++------
 sorc/link_workflow.sh    | 18 ++++++++----------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index b6db132156..71add2bd7b 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -32,22 +32,22 @@ module use "${_HOMEgfs}/sorc/ufs_model.fd/modulefiles"
 module load "ufs_${MACHINE_ID}.intel"
 set -x
 
-#Set WW3 directory, switch, prep and post exes
+#Set WW3 directory
 cd "${_HOMEgfs}/sorc/ufs_model.fd/WW3" || exit 1
 WW3_DIR=$(pwd -P)
 export WW3_DIR
-export SWITCHFILE="${WW3_DIR}/${ww3switch}"
 
 # Determine which switch to use
 if [[ "${PDLIB}" == "ON" ]]; then
   ww3switch="model/bin/switch_meshcap_pdlib"
-  path_build="${WW3_DIR}/build_pdlib"
-  path_install="${WW3_DIR}/install_pdlib"
+  path_build="${WW3_DIR}/build/pdlib_ON"
+  path_install="${WW3_DIR}/install/pdlib_ON"
 else
   ww3switch="model/bin/switch_meshcap"
-  path_build="${WW3_DIR}/build"
-  path_install="install"
+  path_build="${WW3_DIR}/build/pdlib_OFF"
+  path_install="${WW3_DIR}/install/pdlib_OFF"
 fi
+export SWITCHFILE="${WW3_DIR}/${ww3switch}"
 
 #create build directory:
 [[ -d "${path_build}" ]] && rm -rf "${path_build}"
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index ead835eed8..5740f6d65c 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -293,28 +293,26 @@ for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.
 done
 
 [[ -s "gfs_model.x" ]] && rm -f gfs_model.x
-[[ -s "gefs_model.x" ]] && rm -f gefs_model.x
-[[ -s "sfs_model.x" ]] && rm -f sfs_model.x
 if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" .; fi
+[[ -s "gefs_model.x" ]] && rm -f gefs_model.x
 if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" .; fi
+[[ -s "sfs_model.x" ]] && rm -f sfs_model.x
 if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" .; fi
 
 # WW3 pre/post executables
 declare -a ww3_exes=("ww3_grid" "ww3_prep" "ww3_prnc" "ww3_outp" "ww3_outf" "ww3_gint" "ww3_ounf" "ww3_ounp" "ww3_grib")
 # TODO: ww3_prep, ww3_outf, ww3_ounf, ww3_ounp are not used in the workflow # FIXME or remove them from the list
-if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install" ]]; then # installed with PDLIB=OFF (Used in GEFS)
+if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_OFF" ]]; then # installed with PDLIB=OFF (Used in GEFS)
   for ww3exe in "${ww3_exes[@]}"; do
-    [[ -s "gefs_${ww3exe}" ]] && rm -f "gefs_${HOMEgfs}/exec/${ww3exe}"
-    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/bin/${ww3exe}" \
-      "${HOMEgfs}/exec/gefs_${ww3exe}.x"
+    [[ -s "gefs_${ww3exe}.x" ]] && rm -f "gefs_${ww3exe}.x"
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_OFF/bin/${ww3exe}" "${HOMEgfs}/exec/gefs_${ww3exe}.x"
   done
 fi
 
-if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install_pdlib" ]]; then # installed with PDLIB=ON (Used in GFS, may be SFS later)
+if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_ON" ]]; then # installed with PDLIB=ON (Used in GFS, may be SFS later)
   for ww3exe in "${ww3_exes[@]}"; do
-    [[ -s "gfs_${ww3exe}" ]] && rm -f "gfs_${ww3exe}"
-    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install_pdlib/bin/${ww3exe}" \
-      "${HOMEgfs}/exec/gfs_${ww3exe}.x"
+    [[ -s "gfs_${ww3exe}.x" ]] && rm -f "gfs_${ww3exe}.x"
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_ON/bin/${ww3exe}" "${HOMEgfs}/exec/gfs_${ww3exe}.x"
   done
 fi
 

From 612ef35493de703c1d982d83b1b061404d6779da Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Fri, 6 Dec 2024 13:46:01 -0500
Subject: [PATCH 10/37] update Jenkinsfile to build in a single location and
 use it

---
 ci/Jenkinsfile                       | 59 +++++++++++-----------------
 ci/cases/yamls/build.yaml            |  3 --
 ci/scripts/utils/ci_utils_wrapper.sh |  4 +-
 3 files changed, 25 insertions(+), 41 deletions(-)
 delete mode 100644 ci/cases/yamls/build.yaml

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index b3bd6a917a..092d5dc535 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -1,7 +1,8 @@
 def Machine = 'none'
 def machine = 'none'
 def CUSTOM_WORKSPACE = 'none'
-def cases = ''
+def HOMEgfs = 'none'
+def CI_CASES = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
@@ -19,8 +20,8 @@ pipeline {
     }
 
     stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
-             // which is used to designate the Nodes in the Jenkins Controler by the agent label
-             // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+             // which is used to designate the Nodes in the Jenkins Controller by the agent label
+             // Each Jenkins Node is connected to said machine via an JAVA agent via an ssh tunnel
              // no op 2
 
         stage('1. Get Machine') {
@@ -95,7 +96,7 @@ pipeline {
                 axes {
                     axis {
                         name 'system'
-                        values 'gfs', 'gefs'
+                        values 'gfs'
                     }
                 }
                 stages {
@@ -103,8 +104,7 @@ pipeline {
                         steps {
                           catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                             script {
-                                def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
-                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
+                                HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow"
                                 sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
@@ -115,7 +115,7 @@ pipeline {
                                         }
                                     } else {
                                         try {
-                                            echo "Checking out the code for ${system} on ${Machine} using scm in ${HOMEgfs}"
+                                            echo "Checking out the code on ${Machine} using scm in ${HOMEgfs}"
                                             checkout scm
                                         } catch (Exception e) {
                                             if (env.CHANGE_ID) {
@@ -127,14 +127,13 @@ pipeline {
                                         def gist_url = ""
                                         def error_logs = ""
                                         def error_logs_message = ""
-                                        def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
-                                        def build_args_list = builds_file['builds']
-                                        def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
                                         dir("${HOMEgfs}/sorc") {
                                             try {
-                                                sh(script: "${build_args}")
+                                                sh(script: './build_all.sh -kgu')  // build the global-workflow executables for GFS variant (UFS-wx-model, WW3 pre/post executables)
+                                                sh(script: './build_ww3prepost.sh -w')  // build the WW3 pre/post processing executables for GEFS variant
+                                                sh(script: './build_ufs.sh -w')  // build the UFS-wx-model executable for GEFS variant
                                             } catch (Exception error_build) {
-                                                echo "Failed to build system: ${error_build.getMessage()}"
+                                                echo "Failed to build global-workflow: ${error_build.getMessage()}"
                                                 if ( fileExists("logs/error.logs") ) {
                                                     def fileContent = readFile 'logs/error.logs'
                                                     def lines = fileContent.readLines()
@@ -157,24 +156,22 @@ pipeline {
                                                         echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                                     }
                                                     STATUS = 'Failed'
-                                                    error("Failed to build system on ${Machine}")
+                                                    error("Failed to build global-workflow on ${Machine}")
                                                 }
                                             }
                                             sh(script: './link_workflow.sh')
-                                            //sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
-                                    if (env.CHANGE_ID && system == 'gfs') {
+                                    if (env.CHANGE_ID) {
                                         try {
                                             sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
                                         } catch (Exception e) {
                                             echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
                                         }
                                     }
-                                    if (system == 'gfs') {
-                                        cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
-                                        echo "Cases to run: ${cases}"
-                                    }
+                                    // Get a list of CI cases to run
+                                    CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                    echo "Cases to run: ${CI_CASES}"
                                }
                             }
                           }
@@ -191,21 +188,16 @@ pipeline {
             agent { label NodeName[machine].toLowerCase() }
             steps {
                 script {
-                    def parallelStages = cases.collectEntries { caseName ->
+                    def parallelStages = CI_CASES.collectEntries { caseName ->
                         ["${caseName}": {
                             stage("Create ${caseName}") {
                                 script {
-                                        sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
-                                        def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                        def build_system = yaml_case.experiment.system
-                                        def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                        env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                         env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
                                         try {
                                               error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
                                         } catch (Exception error_create) {
-                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
-                                            error("Case ${caseName} failed to create experment directory")
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
+                                            error("Case ${caseName} failed to create experiment directory")
                                         }
                                      }
                             }
@@ -213,15 +205,11 @@ pipeline {
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
-                                                HOMEgfs = "${CUSTOM_WORKSPACE}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
-                                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                                 def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
-                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                                def build_system = yaml_case.experiment.system
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'")
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
@@ -251,7 +239,7 @@ pipeline {
                                                                 echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                                             }
                                                         } else {
-                                                            echo "No error logs found for failed cases in $CUSTOM_WORKSPACE/RUNTESTS/${pslot}_error.logs"
+                                                            echo "No error logs found for failed cases in ${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                         }
                                                         STATUS = 'Failed'
                                                         try {
@@ -273,11 +261,10 @@ pipeline {
             }
         }
 
-        stage( '5. FINALIZE' ) {
+        stage( '5. Finalize' ) {
             agent { label NodeName[machine].toLowerCase() }
             steps {
                 script {
-                    env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                     sh(script: """
                         labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
                         for label in \$labels; do
@@ -298,7 +285,7 @@ pipeline {
                             echo "Failed to remove custom work directory ${CUSTOM_WORKSPACE} on ${Machine}: ${e.getMessage()}"
                         }
                     } else {
-                        echo "Failed to build and run Global-workflow in ${CUSTOM_WORKSPACE} and on ${Machine}"
+                        echo "Failed to build and run global-workflow in ${CUSTOM_WORKSPACE} on ${Machine}"
                     }
                 }
             }
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
deleted file mode 100644
index 2b2938719f..0000000000
--- a/ci/cases/yamls/build.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-builds:
- - gefs: './build_all.sh -kw'
- - gfs: './build_all.sh -kgu'
diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh
index 51c392fb99..a4fd1776c8 100755
--- a/ci/scripts/utils/ci_utils_wrapper.sh
+++ b/ci/scripts/utils/ci_utils_wrapper.sh
@@ -3,7 +3,7 @@
 HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
 source "${HOMEgfs}/ush/detect_machine.sh"
 
-utitilty_function="${1}"
+utility_function="${1}"
 
 source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
-${utitilty_function} "${@:2}"
+${utility_function} "${@:2}"

From 080186ad12f280a7da189909ecb6441e44141be0 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Fri, 6 Dec 2024 13:58:58 -0500
Subject: [PATCH 11/37] remove build matrix from Jenkinsfile

---
 ci/Jenkinsfile | 144 +++++++++++++++++++++++--------------------------
 1 file changed, 66 insertions(+), 78 deletions(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 092d5dc535..c3f2197e06 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -4,7 +4,7 @@ def CUSTOM_WORKSPACE = 'none'
 def HOMEgfs = 'none'
 def CI_CASES = ''
 def GH = 'none'
-// Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
+// Location of the custom workspaces for each machine in the CI system.  They are persistent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
 def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
 def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
@@ -91,90 +91,78 @@ pipeline {
         }
 
         stage('3. Build System') {
-            matrix {
-                agent { label NodeName[machine].toLowerCase() }
-                axes {
-                    axis {
-                        name 'system'
-                        values 'gfs'
-                    }
-                }
-                stages {
-                    stage('Building') {
-                        steps {
-                          catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                            script {
-                                HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow"
-                                sh(script: "mkdir -p ${HOMEgfs}")
-                                ws(HOMEgfs) {
-                                    if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
-                                        sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
-                                        checkout scm
-                                        dir('sorc') {
-                                            sh(script: './link_workflow.sh')
-                                        }
-                                    } else {
-                                        try {
-                                            echo "Checking out the code on ${Machine} using scm in ${HOMEgfs}"
-                                            checkout scm
-                                        } catch (Exception e) {
-                                            if (env.CHANGE_ID) {
-                                                sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine} in Build# ${env.BUILD_NUMBER}: ${e.getMessage()}" """)
-                                            }
-                                            STATUS = 'Failed'
-                                            error("Failed to checkout: ${e.getMessage()}")
-                                        }
-                                        def gist_url = ""
-                                        def error_logs = ""
-                                        def error_logs_message = ""
-                                        dir("${HOMEgfs}/sorc") {
-                                            try {
-                                                sh(script: './build_all.sh -kgu')  // build the global-workflow executables for GFS variant (UFS-wx-model, WW3 pre/post executables)
-                                                sh(script: './build_ww3prepost.sh -w')  // build the WW3 pre/post processing executables for GEFS variant
-                                                sh(script: './build_ufs.sh -w')  // build the UFS-wx-model executable for GEFS variant
-                                            } catch (Exception error_build) {
-                                                echo "Failed to build global-workflow: ${error_build.getMessage()}"
-                                                if ( fileExists("logs/error.logs") ) {
-                                                    def fileContent = readFile 'logs/error.logs'
-                                                    def lines = fileContent.readLines()
-                                                    for (line in lines) {
-                                                        echo "archiving: ${line}"
-                                                        if (fileExists("${line}") && readFile("${line}").length() > 0 ){
-                                                            try {
-                                                                archiveArtifacts artifacts: "${line}", fingerprint: true
-                                                                error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
-                                                                error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
-                                                            }
-                                                            catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
-                                                        }
-                                                    }
+            agent { label NodeName[machine].toLowerCase() }
+            steps {
+                catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                    script {
+                        HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow"
+                        sh(script: "mkdir -p ${HOMEgfs}")
+                        ws(HOMEgfs) {
+                            if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
+                                sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
+                                checkout scm
+                                dir('sorc') {
+                                    sh(script: './link_workflow.sh')
+                                }
+                            } else {
+                                try {
+                                    echo "Checking out the code on ${Machine} using scm in ${HOMEgfs}"
+                                    checkout scm
+                                } catch (Exception e) {
+                                    if (env.CHANGE_ID) {
+                                        sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine} in Build# ${env.BUILD_NUMBER}: ${e.getMessage()}" """)
+                                    }
+                                    STATUS = 'Failed'
+                                    error("Failed to checkout: ${e.getMessage()}")
+                                }
+                                def gist_url = ""
+                                def error_logs = ""
+                                def error_logs_message = ""
+                                dir("${HOMEgfs}/sorc") {
+                                    try {
+                                        sh(script: './build_all.sh -kgu')  // build the global-workflow executables for GFS variant (UFS-wx-model, WW3 pre/post executables)
+                                        sh(script: './build_ww3prepost.sh -w')  // build the WW3 pre/post processing executables for GEFS variant
+                                        sh(script: './build_ufs.sh -w')  // build the UFS-wx-model executable for GEFS variant
+                                    } catch (Exception error_build) {
+                                        echo "Failed to build global-workflow: ${error_build.getMessage()}"
+                                        if ( fileExists("logs/error.logs") ) {
+                                            def fileContent = readFile 'logs/error.logs'
+                                            def lines = fileContent.readLines()
+                                            for (line in lines) {
+                                                echo "archiving: ${line}"
+                                                if (fileExists("${line}") && readFile("${line}").length() > 0 ){
                                                     try {
-                                                        sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
-                                                        gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
-                                                        sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
-                                                    } catch (Exception error_comment) {
-                                                        echo "Failed to comment on PR: ${error_comment.getMessage()}"
+                                                        archiveArtifacts artifacts: "${line}", fingerprint: true
+                                                        error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
+                                                        error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
                                                     }
-                                                    STATUS = 'Failed'
-                                                    error("Failed to build global-workflow on ${Machine}")
+                                                    catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
                                                 }
                                             }
-                                            sh(script: './link_workflow.sh')
-                                        }
-                                    }
-                                    if (env.CHANGE_ID) {
-                                        try {
-                                            sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
-                                        } catch (Exception e) {
-                                            echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
+                                            try {
+                                                sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+                                                gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+                                                sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+                                            } catch (Exception error_comment) {
+                                                echo "Failed to comment on PR: ${error_comment.getMessage()}"
+                                            }
+                                            STATUS = 'Failed'
+                                            error("Failed to build global-workflow on ${Machine}")
                                         }
                                     }
-                                    // Get a list of CI cases to run
-                                    CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
-                                    echo "Cases to run: ${CI_CASES}"
-                               }
+                                    sh(script: './link_workflow.sh')
+                                }
+                            }
+                            if (env.CHANGE_ID) {
+                                try {
+                                    sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """)
+                                } catch (Exception e) {
+                                    echo "Failed to update label from Building to Running: ${e.getMessage()}"
+                                }
                             }
-                          }
+                            // Get a list of CI cases to run
+                            CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                            echo "Cases to run: ${CI_CASES}"
                         }
                     }
                 }

From f8e3d6ada689c809302fe8f4f910cb45a7bb934f Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Fri, 6 Dec 2024 14:10:18 -0500
Subject: [PATCH 12/37] fix shellnorm found issue

---
 sorc/build_ww3prepost.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index 71add2bd7b..28f3261608 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 set -x
 
-readonly _HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}")")/.." && pwd -P)
+_HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P)
 cd "${_HOMEgfs}/sorc" || exit 1
 
 # Default settings

From fc41b6d4c94efddd7f89883bcf88d6ef240afed7 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Fri, 6 Dec 2024 14:14:01 -0500
Subject: [PATCH 13/37] fix shellnorm found issue by disabling it

---
 sorc/build_ww3prepost.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index 28f3261608..69499d4c58 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -1,6 +1,7 @@
 #! /usr/bin/env bash
 set -x
 
+# shellcheck disable=SC2312
 _HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P)
 cd "${_HOMEgfs}/sorc" || exit 1
 

From c61ff88bc68f17370edb289060296735ea538bca Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Fri, 6 Dec 2024 14:17:25 -0500
Subject: [PATCH 14/37] fix not changed shellcheck detected error

---
 ush/wave_outp_spec.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh
index ae818171ca..9561250f96 100755
--- a/ush/wave_outp_spec.sh
+++ b/ush/wave_outp_spec.sh
@@ -219,7 +219,7 @@ source "${USHgfs}/preamble.sh"
     set +x
     echo ' '
     echo '***************************************************************** '
-    echo '*** FATAL ERROR : OUTPUT DATA FILE FOR BOUY $bouy NOT FOUND *** '
+    echo "*** FATAL ERROR : OUTPUT DATA FILE FOR BOUY ${bouy} NOT FOUND *** "
     echo '***************************************************************** '
     echo ' '
     set_trace

From 200d6bdcdc378773fefa1ac5c13aacde4f7379a8 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Fri, 6 Dec 2024 15:11:22 -0500
Subject: [PATCH 15/37] add comment back in ww3prepost build script

---
 sorc/build_ww3prepost.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index 69499d4c58..7b3537db84 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -54,6 +54,7 @@ export SWITCHFILE="${WW3_DIR}/${ww3switch}"
 [[ -d "${path_build}" ]] && rm -rf "${path_build}"
 mkdir -p "${path_build}" || exit 1
 cd "${path_build}" || exit 1
+echo "Forcing a SHRD build"
 
 buildswitch="${path_build}/switch"
 

From 37d1fb943f9206002820814dadb51d83ab1211fa Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:48:31 -0500
Subject: [PATCH 16/37] Update ush/wave_grid_interp_sbs.sh

fix for wht_OK
---
 ush/wave_grid_interp_sbs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh
index 190dff221b..25a446385a 100755
--- a/ush/wave_grid_interp_sbs.sh
+++ b/ush/wave_grid_interp_sbs.sh
@@ -144,7 +144,7 @@ source "${USHgfs}/preamble.sh"
   export err=$?;err_chk
 
 # Write interpolation file to main TEMP dir area if not there yet
-  if [ "wht_OK" = 'no' ]  # FIXME: This is never going to evaluate to true, wht_OK is a string and needs to be ${wht_OK}
+  if [ "${wht_OK}" = 'no' ]
   then
     cp -f ./WHTGRIDINT.bin ${DATA}/ww3_gint.WHTGRIDINT.bin.${grdID}
     cp -f ./WHTGRIDINT.bin ${FIXgfs}/wave/ww3_gint.WHTGRIDINT.bin.${grdID}

From ab66442f1f9d5f6ec9a70e07e5fb359803728206 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 10 Dec 2024 12:41:08 -0500
Subject: [PATCH 17/37] Update sorc/link_workflow.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 sorc/link_workflow.sh | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 5740f6d65c..7c894ef298 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -292,12 +292,16 @@ for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" .
 done
 
-[[ -s "gfs_model.x" ]] && rm -f gfs_model.x
-if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gfs_model.x" .; fi
-[[ -s "gefs_model.x" ]] && rm -f gefs_model.x
-if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/gefs_model.x" .; fi
-[[ -s "sfs_model.x" ]] && rm -f sfs_model.x
-if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/sfs_model.x" .; fi
+declare -a model_systems=("gfs" "gefs" "sfs")
+for sys in "${model_systems[@]}"; do
+  model_exe="${sys}_model.x"
+  if [[ -s "${model_exe}" ]]; then
+    rm -f "${model_exe}"
+  fi
+  if [[ -f "${HOMEgfs}/sorc/ufs_model.fd/tests/${model_exe}" ]]; then
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/${model_exe}" "${model_exe}"
+  fi
+done
 
 # WW3 pre/post executables
 declare -a ww3_exes=("ww3_grid" "ww3_prep" "ww3_prnc" "ww3_outp" "ww3_outf" "ww3_gint" "ww3_ounf" "ww3_ounp" "ww3_grib")

From dae774ab0fc401b0a9d664e0c907db41b9e3056e Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 10 Dec 2024 12:41:55 -0500
Subject: [PATCH 18/37] Update ush/wave_grid_interp_sbs.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ush/wave_grid_interp_sbs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh
index 25a446385a..5420d405c8 100755
--- a/ush/wave_grid_interp_sbs.sh
+++ b/ush/wave_grid_interp_sbs.sh
@@ -134,7 +134,7 @@ source "${USHgfs}/preamble.sh"
 # 1.b Run interpolation code
 
   export pgm="${NET,,}_ww3_gint.x"
-  . prep_step
+  source prep_step
 
   set +x
   echo "   Executing ${pgm}"

From 1d47c9eb11e7feb70f7805893c6acec5df50527f Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 10 Dec 2024 12:42:02 -0500
Subject: [PATCH 19/37] Update ush/wave_prnc_cur.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ush/wave_prnc_cur.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_prnc_cur.sh b/ush/wave_prnc_cur.sh
index b084b8dcac..cfd499330f 100755
--- a/ush/wave_prnc_cur.sh
+++ b/ush/wave_prnc_cur.sh
@@ -75,7 +75,7 @@ ${NLN} "cur_glo_uv_${PDY}_${fext}${fh3}_5min.nc" "cur.nc"
 ${NLN} "${DATA}/mod_def.${WAVECUR_FID}" ./mod_def.ww3
 
 export pgm="${NET,,}_ww3_prnc.x"
-. prep_step
+source prep_step
 
 "${EXECgfs}/${pgm}" 1> prnc_${WAVECUR_FID}_${ymdh_rtofs}.out 2>&1
 export err=$?; err_chk

From b136b8a8c7d1b853581b799823823408ed37aaf5 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 10 Dec 2024 12:42:11 -0500
Subject: [PATCH 20/37] Update ush/wave_prnc_ice.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ush/wave_prnc_ice.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_prnc_ice.sh b/ush/wave_prnc_ice.sh
index 890df71973..d812bccd10 100755
--- a/ush/wave_prnc_ice.sh
+++ b/ush/wave_prnc_ice.sh
@@ -143,7 +143,7 @@ source "${USHgfs}/preamble.sh"
   cp -f ${DATA}/ww3_prnc.ice.$WAVEICE_FID.inp.tmpl ww3_prnc.inp
 
   export pgm="${NET,,}_ww3_prnc.x"
-  . prep_step
+  source prep_step
 
   "${EXECgfs}/${pgm}" 1> prnc_${WAVEICE_FID}_${cycle}.out 2>&1
   export err=$?; err_chk

From ebf617653c830afec917fefaaea5a67c0e6c3c8e Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 10 Dec 2024 12:43:07 -0500
Subject: [PATCH 21/37] Update scripts/exgfs_wave_post_pnt.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 scripts/exgfs_wave_post_pnt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh
index 9b40852092..96c60fc96c 100755
--- a/scripts/exgfs_wave_post_pnt.sh
+++ b/scripts/exgfs_wave_post_pnt.sh
@@ -273,7 +273,7 @@ source "${USHgfs}/preamble.sh"
     ${NLN} ./mod_def.${waveuoutpGRD} ./mod_def.ww3
 
     export pgm="${NET,,}_ww3_outp.x"
-    . prep_step
+    source prep_step
 
     "${EXECgfs}/${pgm}" > buoy_lst.loc 2>&1
     export err=$?;err_chk

From 93ad4bf93e6dec66ffcf867e40b25fb64e5fbafe Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 10 Dec 2024 12:43:14 -0500
Subject: [PATCH 22/37] Update ush/wave_outp_spec.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 ush/wave_outp_spec.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh
index 9561250f96..929979bbf9 100755
--- a/ush/wave_outp_spec.sh
+++ b/ush/wave_outp_spec.sh
@@ -162,7 +162,7 @@ source "${USHgfs}/preamble.sh"
 # 2.b Run the postprocessor
 
   export pgm="${NET,,}_ww3_outp.x"
-  . prep_step
+  source prep_step
 
   set +x
   echo "   Executing ${EXECgfs}/${pgm}"

From b5ede8bac175441aaad0bc93897194f08600d277 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 09:40:07 -0500
Subject: [PATCH 23/37] capture stdout/stderr from Jenkinsbuild for gefs
 variants.  Also, was missing the name of the GEFS variant model executable
 that caused the gefs test to fail

---
 ci/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index c3f2197e06..a6fd9ed906 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -121,8 +121,8 @@ pipeline {
                                 dir("${HOMEgfs}/sorc") {
                                     try {
                                         sh(script: './build_all.sh -kgu')  // build the global-workflow executables for GFS variant (UFS-wx-model, WW3 pre/post executables)
-                                        sh(script: './build_ww3prepost.sh -w')  // build the WW3 pre/post processing executables for GEFS variant
-                                        sh(script: './build_ufs.sh -w')  // build the UFS-wx-model executable for GEFS variant
+                                        sh(script: './build_ww3prepost.sh -w > ./logs/build_ww3prepost_gefs.log 2>&1')  // build the WW3 pre/post processing executables for GEFS variant
+                                        sh(script: './build_ufs.sh -w -e gefs_model.x > ./logs/build_ufs_gefs.log 2>&1')  // build the UFS-wx-model executable for GEFS variant
                                     } catch (Exception error_build) {
                                         echo "Failed to build global-workflow: ${error_build.getMessage()}"
                                         if ( fileExists("logs/error.logs") ) {

From 4531678e1ec55a11a451482a25cb862e84b206aa Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 15:10:56 -0500
Subject: [PATCH 24/37] Update sorc/link_workflow.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 sorc/link_workflow.sh | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 7c894ef298..fb76d66d30 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -306,19 +306,23 @@ done
 # WW3 pre/post executables
 declare -a ww3_exes=("ww3_grid" "ww3_prep" "ww3_prnc" "ww3_outp" "ww3_outf" "ww3_gint" "ww3_ounf" "ww3_ounp" "ww3_grib")
 # TODO: ww3_prep, ww3_outf, ww3_ounf, ww3_ounp are not used in the workflow # FIXME or remove them from the list
-if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_OFF" ]]; then # installed with PDLIB=OFF (Used in GEFS)
-  for ww3exe in "${ww3_exes[@]}"; do
-    [[ -s "gefs_${ww3exe}.x" ]] && rm -f "gefs_${ww3exe}.x"
-    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_OFF/bin/${ww3exe}" "${HOMEgfs}/exec/gefs_${ww3exe}.x"
-  done
-fi
-
-if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_ON" ]]; then # installed with PDLIB=ON (Used in GFS, may be SFS later)
-  for ww3exe in "${ww3_exes[@]}"; do
-    [[ -s "gfs_${ww3exe}.x" ]] && rm -f "gfs_${ww3exe}.x"
-    ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/pdlib_ON/bin/${ww3exe}" "${HOMEgfs}/exec/gfs_${ww3exe}.x"
-  done
-fi
+declare -A wave_systems
+wave_systems["gfs"]="pdlib_ON"
+wave_systems["gefs"]="pdlib_OFF"
+wave_systems["sfs"]="pdlib_OFF"
+
+for sys in ${!wave_systems[@]}; do
+  build_loc="${wave_systems[${sys}]}"
+  if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/${build_loc}" ]]; then
+    for ww3exe in "${ww3_exes[@]}"; do
+      target_ww3_exe="${sys}_${ww3exe}.x"
+      if [[ -s "${target_ww3_exe}" ]]; then
+        rm -f "${target_ww3_exe}"
+      fi
+      ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/${build_loc}/bin/${ww3exe}.x" "${HOMEgfs}/exec/${target_ww3_exe}"
+    done
+  fi
+done
 
 [[ -s "upp.x" ]] && rm -f upp.x
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/exec/upp.x" .

From 842e71777259c269f70305510e7ec878f77eff77 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 15:15:27 -0500
Subject: [PATCH 25/37] Update sorc/build_ufs.sh

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 sorc/build_ufs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh
index 863ca30b65..773c104be3 100755
--- a/sorc/build_ufs.sh
+++ b/sorc/build_ufs.sh
@@ -8,7 +8,7 @@ APP="S2SWA"
 CCPP_SUITES="FV3_GFS_v17_p8_ugwpv1,FV3_GFS_v17_coupled_p8_ugwpv1,FV3_global_nest_v1"  # TODO: does the g-w need to build with all these CCPP_SUITES?
 PDLIB="ON"
 HYDRO="OFF"
-EXEC_NAME="ufs_model.x"
+EXEC_NAME="gfs_model.x"
 
 while getopts ":da:fj:e:vwy" option; do
   case "${option}" in

From c1e01f8c804302b76e322e4f6fd16718533122f4 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 15:16:02 -0500
Subject: [PATCH 26/37] Update sorc/link_workflow.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 sorc/link_workflow.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index fb76d66d30..234294b23d 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -311,7 +311,7 @@ wave_systems["gfs"]="pdlib_ON"
 wave_systems["gefs"]="pdlib_OFF"
 wave_systems["sfs"]="pdlib_OFF"
 
-for sys in ${!wave_systems[@]}; do
+for sys in "${!wave_systems[@]}"; do
   build_loc="${wave_systems[${sys}]}"
   if [[ -d "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/${build_loc}" ]]; then
     for ww3exe in "${ww3_exes[@]}"; do

From d3e76a6f6f7c5f6dbf0b21050a355c5f28c10177 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 16:48:18 -0500
Subject: [PATCH 27/37] Update sorc/link_workflow.sh

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 sorc/link_workflow.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 234294b23d..8c1c0658e7 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -319,7 +319,7 @@ for sys in "${!wave_systems[@]}"; do
       if [[ -s "${target_ww3_exe}" ]]; then
         rm -f "${target_ww3_exe}"
       fi
-      ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/${build_loc}/bin/${ww3exe}.x" "${HOMEgfs}/exec/${target_ww3_exe}"
+      ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/WW3/install/${build_loc}/bin/${ww3exe}" "${HOMEgfs}/exec/${target_ww3_exe}"
     done
   fi
 done

From be0ca2c33b93701e03677d253d1155037940be93 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 17:04:57 -0500
Subject: [PATCH 28/37] since using single build location, GEFS wcoss testing
 can be enabled

---
 ci/cases/pr/C48_S2SWA_gefs.yaml            | 2 +-
 ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
index f39031f1a1..856759b46a 100644
--- a/ci/cases/pr/C48_S2SWA_gefs.yaml
+++ b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -18,4 +18,4 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_defaults_ci.yaml
 
 skip_ci_on_hosts:
-  - wcoss2
+  - None
diff --git a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml
index 7118dde53f..b5c2742406 100644
--- a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml
+++ b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml
@@ -19,4 +19,4 @@ arguments:
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610
 
 skip_ci_on_hosts:
-  - wcoss2
+  - None

From c37409d842a1d6539e2dbe316f362b3de1d99c85 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 11 Dec 2024 17:36:26 -0500
Subject: [PATCH 29/37] update Jenkinsfile to try to do scm checkout if it
 fails the first time;

---
 ci/Jenkinsfile | 89 +++++++++++++++++++++++++-------------------------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index a6fd9ed906..9e2381268d 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -82,7 +82,9 @@ pipeline {
                         properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim()
                         CUSTOM_WORKSPACE = "${WORKSPACE}"
-                        sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*")
+                        HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow"
+                        sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/global-workflow; mkdir -p ${CUSTOM_WORKSPACE}/global-workflow")
+                        sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS; mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)
                     }
                     echo "Building and running on ${Machine} in directory ${CUSTOM_WORKSPACE}"
@@ -95,63 +97,60 @@ pipeline {
             steps {
                 catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                     script {
-                        HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow"
-                        sh(script: "mkdir -p ${HOMEgfs}")
                         ws(HOMEgfs) {
-                            if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
-                                sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
+                            echo "Checking out the code on ${Machine} using scm in ${HOMEgfs}"
+                            try {
                                 checkout scm
-                                dir('sorc') {
-                                    sh(script: './link_workflow.sh')
-                                }
-                            } else {
+                            } catch (Exception e) {
+                                echo "Failed to checkout the code on ${Machine} using scm in ${HOMEgfs}, try again ..."
+                                sleep time: 45, unit: 'SECONDS'
                                 try {
-                                    echo "Checking out the code on ${Machine} using scm in ${HOMEgfs}"
                                     checkout scm
-                                } catch (Exception e) {
+                                } catch (Exception ee) {
+                                    echo "Failed to checkout the code on ${Machine} using scm in ${HOMEgfs}: ${ee.getMessage()}"
                                     if (env.CHANGE_ID) {
-                                        sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine} in Build# ${env.BUILD_NUMBER}: ${e.getMessage()}" """)
+                                        sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine} in Build# ${env.BUILD_NUMBER}: ${ee.getMessage()}" """)
                                     }
                                     STATUS = 'Failed'
-                                    error("Failed to checkout: ${e.getMessage()}")
+                                    error("Failed to checkout: ${ee.getMessage()}")
                                 }
-                                def gist_url = ""
-                                def error_logs = ""
-                                def error_logs_message = ""
-                                dir("${HOMEgfs}/sorc") {
-                                    try {
-                                        sh(script: './build_all.sh -kgu')  // build the global-workflow executables for GFS variant (UFS-wx-model, WW3 pre/post executables)
-                                        sh(script: './build_ww3prepost.sh -w > ./logs/build_ww3prepost_gefs.log 2>&1')  // build the WW3 pre/post processing executables for GEFS variant
-                                        sh(script: './build_ufs.sh -w -e gefs_model.x > ./logs/build_ufs_gefs.log 2>&1')  // build the UFS-wx-model executable for GEFS variant
-                                    } catch (Exception error_build) {
-                                        echo "Failed to build global-workflow: ${error_build.getMessage()}"
-                                        if ( fileExists("logs/error.logs") ) {
-                                            def fileContent = readFile 'logs/error.logs'
-                                            def lines = fileContent.readLines()
-                                            for (line in lines) {
-                                                echo "archiving: ${line}"
-                                                if (fileExists("${line}") && readFile("${line}").length() > 0 ){
-                                                    try {
-                                                        archiveArtifacts artifacts: "${line}", fingerprint: true
-                                                        error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
-                                                        error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
-                                                    }
-                                                    catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
+                            }
+                            def gist_url = ""
+                            def error_logs = ""
+                            def error_logs_message = ""
+                            dir("${HOMEgfs}/sorc") {
+                                try {
+                                    sh(script: './build_all.sh -kgu')  // build the global-workflow executables for GFS variant (UFS-wx-model, WW3 pre/post executables)
+                                    sh(script: './build_ww3prepost.sh -w > ./logs/build_ww3prepost_gefs.log 2>&1')  // build the WW3 pre/post processing executables for GEFS variant
+                                    sh(script: './build_ufs.sh -w -e gefs_model.x > ./logs/build_ufs_gefs.log 2>&1')  // build the UFS-wx-model executable for GEFS variant
+                                } catch (Exception error_build) {
+                                    echo "Failed to build global-workflow: ${error_build.getMessage()}"
+                                    if ( fileExists("logs/error.logs") ) {
+                                        def fileContent = readFile 'logs/error.logs'
+                                        def lines = fileContent.readLines()
+                                        for (line in lines) {
+                                            echo "archiving: ${line}"
+                                            if (fileExists("${line}") && readFile("${line}").length() > 0 ){
+                                                try {
+                                                    archiveArtifacts artifacts: "${line}", fingerprint: true
+                                                    error_logs = error_logs + "${HOMEgfs}/sorc/${line} "
+                                                    error_logs_message = error_logs_message + "${HOMEgfs}/sorc/${line}\n"
                                                 }
+                                                catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" }
                                             }
-                                            try {
-                                                sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
-                                                gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
-                                                sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
-                                            } catch (Exception error_comment) {
-                                                echo "Failed to comment on PR: ${error_comment.getMessage()}"
-                                            }
-                                            STATUS = 'Failed'
-                                            error("Failed to build global-workflow on ${Machine}")
                                         }
+                                        try {
+                                            sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
+                                            gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+                                            sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+                                        } catch (Exception error_comment) {
+                                            echo "Failed to comment on PR: ${error_comment.getMessage()}"
+                                        }
+                                        STATUS = 'Failed'
+                                        error("Failed to build global-workflow on ${Machine}")
                                     }
-                                    sh(script: './link_workflow.sh')
                                 }
+                                sh(script: './link_workflow.sh')
                             }
                             if (env.CHANGE_ID) {
                                 try {

From 50794488eb20d039d4f5c42df006ac4034be9e60 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 12 Dec 2024 09:42:07 -0500
Subject: [PATCH 30/37] undo the fix and reintroduce the incorrect logic. 
 Fixing the bug opens another bug that needs evaluation on the purpose of the
 logic. A FIXME tag has been added

---
 ush/wave_grid_interp_sbs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh
index 5420d405c8..d56c5d0763 100755
--- a/ush/wave_grid_interp_sbs.sh
+++ b/ush/wave_grid_interp_sbs.sh
@@ -144,7 +144,7 @@ source "${USHgfs}/preamble.sh"
   export err=$?;err_chk
 
 # Write interpolation file to main TEMP dir area if not there yet
-  if [ "${wht_OK}" = 'no' ]
+  if [ "wht_OK" = 'no' ]  # FIXME: This is never going to evaluate to true, wht_OK is a string and needs to be ${wht_OK}.  With ${wht_OK}, the next line is trying to copy into ${FIXgfs} space.  This leads to a Permission denied error. The logic here needs to be evaluated and recoded.  #TODO
   then
     cp -f ./WHTGRIDINT.bin ${DATA}/ww3_gint.WHTGRIDINT.bin.${grdID}
     cp -f ./WHTGRIDINT.bin ${FIXgfs}/wave/ww3_gint.WHTGRIDINT.bin.${grdID}

From 41f20a4c79624ada78c2c4acc917ac272f8a6a1c Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 13 Dec 2024 17:35:37 +0000
Subject: [PATCH 31/37] Simplify build_all.sh

---
 sorc/build_all.sh | 232 +++++++++++++++++++++++-----------------------
 1 file changed, 117 insertions(+), 115 deletions(-)

diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 9414846f2a..89c5fdf0bf 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -13,32 +13,28 @@ set +x
 #------------------------------------
 function _usage() {
   cat << EOF
-Builds all of the global-workflow components by calling the individual build
-  scripts in sequence.
+Builds all of the global-workflow components by calling the individual build scripts in parallel.
 
-Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-j n][-v][-w][-y]
+Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-j n][-v] [gfs] [gefs] [sfs] [gsi] [gdas]
   -a UFS_app:
-    Build a specific UFS app instead of the default
+    Build a specific UFS app instead of the default.  This will be applied to all UFS (GFS, GEFS, SFS) builds.
+  -c:
+    Submit the build jobs to compute nodes
   -d:
     Build in debug mode
   -f:
-    Build the UFS model using the -DFASTER=ON option
-  -g:
-    Build GSI
+    Build the UFS model(s) using the -DFASTER=ON option.
   -h:
     Print this help message and exit
-  -j:
-    Specify maximum number of build jobs (n)
   -k:
     Kill all builds if any build fails
-  -u:
-    Build UFS-DA
   -v:
     Execute all build scripts with -v option to turn on verbose where supported
-  -w:
-    Use structured wave grid
-  -y:
-    Use hydrostatic version of FV3
+  -A:
+    HPC account to use for the compute-node builds
+    (default is \$HOMEgfs/ci/platforms/config.\$machine:\$HPC_ACCOUNT)
+
+  Specified systems (gfs, gefs, sfs, gsi, gdas) are non-exclusive, so they can be built together.
 EOF
   exit 1
 }
@@ -48,30 +44,24 @@ readonly HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.."
 cd "${HOMEgfs}/sorc" || exit 1
 
 _build_ufs_opt=""
-_build_ufsda="NO"
-_build_gsi="NO"
 _build_debug=""
 _verbose_opt=""
-_wave_opt=""
-_hydro_opt=""
 _build_job_max=20
 _quick_kill="NO"
-_ufs_exec="-e gfs_model.x"
+_compute_build="NO"
+_hpc_account="default"
 # Reset option counter in case this script is sourced
 OPTIND=1
-while getopts ":a:dfghj:kuvwy" option; do
+while getopts ":a:cdfhj:kA:v" option; do
   case "${option}" in
     a) _build_ufs_opt+="-a ${OPTARG} ";;
+    c) _compute_build="YES" ;;
     f) _build_ufs_opt+="-f ";;
     d) _build_debug="-d" ;;
-    g) _build_gsi="YES" ;;
     h) _usage;;
-    j) _build_job_max="${OPTARG} ";;
     k) _quick_kill="YES" ;;
-    u) _build_ufsda="YES" ;;
+    A) _hpc_account="${OPTARG}"
     v) _verbose_opt="-v";;
-    w) _wave_opt="-w"; _ufs_exec="-e gefs_model.x";;
-    y) _hydro_opt="-y"; _ufs_exec="-e sfs_model.x";;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       _usage
@@ -81,22 +71,103 @@ while getopts ":a:dfghj:kuvwy" option; do
       _usage
       ;;
   esac
+  shift $((OPTIND-1))
 done
 
-shift $((OPTIND-1))
+supported_systems=("gfs" "gefs" "sfs" "gsi" "gdas")
+gfs_builds="gfs gfs_utils ufs_utils upp ww3_struct"
+gefs_builds="gefs gfs_utils ufs_utils upp ww3_struct"
+sfs_builds="sfs gfs_utils ufs_utils upp ww3_unstruct"
+gsi_builds="gsi_enkf gsi_monitor gsi_utils"
+gdas_builds="gdas gsi_monitor gsi_utils"
+
+# Builds that will use a whole compute node each
+compute_builds="gfs gefs sfs"
+# Service node builds
+# The GDASApp (for now) needs to be compiled on the service node
+service_builds="gdas gfs_utils ufs_utils upp ww3_struct ww3_unstruct gsi gsi_utils gsi_monitor"
+
+# Jobs per build ("min max")
+declare -A build_jobs build_opts build_scripts
+build_jobs["gfs"]=8
+build_jobs["gefs"]=8
+build_jobs["sfs"]=8
+build_jobs["gdas"]=8
+build_jobs["gsi_enkf"]=2
+build_jobs["gfs_utils"]=1
+build_jobs["ufs_utils"]=1
+build_jobs["ww3_unstruct"]=1
+build_jobs["ww3_struct"]=1
+build_jobs["gsi_utils"]=1
+build_jobs["gsi_monitor"]=1
+build_jobs["gfs_utils"]=1
+build_jobs["upp"]=1
+
+# Establish build options for each job
+build_opts["gfs"]="${wave_opt} ${_build_ufs_opt} ${_verbose_opt} ${_build_debug} ${_gfs_exec}"
+build_opts["gefs"]="${wave_opt} ${_build_ufs_opt} ${_verbose_opt} ${_build_debug} ${_gefs_exec}"
+build_opts["sfs"]="${wave_opt} ${_build_ufs_opt} ${_verbose_opt} ${_build_debug} ${_sfs_exec}"
+build_opts["upp"]="${_build_debug}"
+build_opts["ww3_unstruct"]="${_verbose_opt} ${_build_debug}"
+build_opts["ww3_struct"]="-w ${_verbose_opt} ${_build_debug}"
+build_opts["gdas"]="${_verbose_opt} ${_build_debug}"
+build_opts["ufs_utils"]="${_verbose_opt} ${_build_debug}"
+build_opts["gfs_utils"]="${_verbose_opt} ${_build_debug}"
+build_opts["gsi_utils"]="${_verbose_opt} ${_build_debug}"
+build_opts["gsi_enkf"]="${_verbose_opt} ${_build_debug}"
+build_opts["gsi_monitor"]="${_verbose_opt} ${_build_debug}"
+
+# Set the build script name for each build
+build_scripts["gfs"]="build_ufs.sh"
+build_scripts["gefs"]="build_ufs.sh"
+build_scripts["sfs"]="build_ufs.sh"
+build_scripts["gdas"]="build_gdas.sh"
+build_scripts["gsi_enkf"]="build_gsi_enkf.sh"
+build_scripts["gfs_utils"]="build_gfs_utils.sh"
+build_scripts["ufs_utils"]="build_ufs_utils.sh"
+build_scripts["ww3_unstruct"]="build_ww3_prepost.sh"
+build_scripts["ww3_struct"]="build_ww3_prepost.sh"
+build_scripts["gsi_utils"]="build_gsi_utils.sh"
+build_scripts["gsi_monitor"]="build_gsi_monitor.sh"
+build_scripts["gfs_utils"]="build_gfs_utils.sh"
+build_scripts["upp"]="build_upp.sh"
+
+# Check the requested systems to make sure we can build them
+declare -A builds
+system_count=0
+for system in ${@}; do
+   if [[ " ${supported_systems[*]} " =~ " ${system} " ]];
+      (( system_count += 1 ))
+      build_list_name="${system}_builds"
+      for build in ${!build_list_name}; do
+         builds["${build}"]="yes"
+      done
+   else
+      echo "Unsupported build system: ${system}"
+      _usage
+   fi
+done
 
-logs_dir="${HOMEgfs}/sorc/logs"
-if [[ ! -d "${logs_dir}" ]]; then
-  echo "Creating logs folder"
-  mkdir -p "${logs_dir}" || exit 1
+# If no build systems were selected, build just the gfs
+if [[ ${system_count} -eq 0 ]]; then
+   system_count=1
+   builds["gfs"]="yes"
 fi
 
-# Check final exec folder exists
-if [[ ! -d "${HOMEgfs}/exec" ]]; then
-  echo "Creating ${HOMEgfs}/exec folder"
-  mkdir -p "${HOMEgfs}/exec"
+# Create directories
+mkdir -p "${HOMEgfs}/sorc/logs" "${HOMEgfs}/exec"
+
+# If we are running this on compute nodes, then call compute_build.sh with the list of builds
+
+if [[ "${_compute_build}" == "YES" ]]; then 
+   #shellcheck disable=SC2086
+   compute_build.sh -a "${_hpc_account}" ${!builds[@]}
+   stat=$?
+   exit ${stat}
 fi
 
+# Otherwise, we are building locally
+
 #------------------------------------
 # GET MACHINE
 #------------------------------------
@@ -123,87 +194,18 @@ ERRSCRIPT=${ERRSCRIPT:-'eval [[ $errs = 0 ]]'}
 # shellcheck disable=
 errs=0
 
-declare -A build_jobs
-declare -A build_opts
-
 #------------------------------------
 # Check which builds to do and assign # of build jobs
 #------------------------------------
 
-# Mandatory builds, unless otherwise specified, for the UFS
-big_jobs=0
-build_jobs["ufs"]=8
-big_jobs=$((big_jobs+1))
-build_opts["ufs"]="${_wave_opt} ${_hydro_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug} ${_ufs_exec}"
-
-build_jobs["upp"]=1
-build_opts["upp"]="${_build_debug}"
-
-build_jobs["ufs_utils"]=1
-build_opts["ufs_utils"]="${_verbose_opt} ${_build_debug}"
-
-build_jobs["gfs_utils"]=1
-build_opts["gfs_utils"]="${_verbose_opt} ${_build_debug}"
-
-build_jobs["ww3prepost"]=1
-build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug}"
-
-# Optional DA builds
-if [[ "${_build_ufsda}" == "YES" ]]; then
-   if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" && "${MACHINE_ID}" != "gaea" ]]; then
-      echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}.  Disabling build."
-   else
-      build_jobs["gdas"]=8
-      big_jobs=$((big_jobs+1))
-      build_opts["gdas"]="${_verbose_opt} ${_build_debug}"
-   fi
-fi
-if [[ "${_build_gsi}" == "YES" ]]; then
-   build_jobs["gsi_enkf"]=2
-   build_opts["gsi_enkf"]="${_verbose_opt} ${_build_debug}"
-fi
-if [[ "${_build_gsi}" == "YES" || "${_build_ufsda}" == "YES" ]] ; then
-   build_jobs["gsi_utils"]=1
-   build_opts["gsi_utils"]="${_verbose_opt} ${_build_debug}"
-   build_jobs["gsi_monitor"]=1
-   build_opts["gsi_monitor"]="${_verbose_opt} ${_build_debug}"
-fi
-
-# Go through all builds and adjust CPU counts down if necessary
-requested_cpus=0
-build_list=""
-for build in "${!build_jobs[@]}"; do
-   if [[ -z "${build_list}" ]]; then
-      build_list="${build}"
-   else
-      build_list="${build_list}, ${build}"
-   fi
-   if [[ ${build_jobs[${build}]} -gt ${_build_job_max} ]]; then
-      build_jobs[${build}]=${_build_job_max}
-   fi
-   requested_cpus=$(( requested_cpus + build_jobs[${build}] ))
-done
-
 echo "Building ${build_list}"
 
-# Go through all builds and adjust CPU counts up if possible
-if [[ ${requested_cpus} -lt ${_build_job_max} && ${big_jobs} -gt 0 ]]; then
-   # Add cores to the gdas and ufs build jobs
-   extra_cores=$(( _build_job_max - requested_cpus ))
-   extra_cores=$(( extra_cores / big_jobs ))
-   for build in "${!build_jobs[@]}"; do
-      if [[ "${build}" == "gdas" || "${build}" == "ufs" ]]; then
-         build_jobs[${build}]=$(( build_jobs[${build}] + extra_cores ))
-      fi
-   done
-fi
-
 procs_in_use=0
 declare -A build_ids
 
 check_builds()
 {
-   for chk_build in "${!build_jobs[@]}"; do
+   for chk_build in "${!builds[@]}"; do
       # Check if the build is complete and if so what the status was
       if [[ -n "${build_ids[${chk_build}]+0}" ]]; then
          if ! ps -p "${build_ids[${chk_build}]}" > /dev/null; then
@@ -213,7 +215,7 @@ check_builds()
                echo "build_${chk_build}.sh failed!  Exiting!"
                echo "Check logs/build_${chk_build}.log for details."
                echo "logs/build_${chk_build}.log" > "${HOMEgfs}/sorc/logs/error.logs"
-               for kill_build in "${!build_jobs[@]}"; do
+               for kill_build in "${!builds[@]}"; do
                   if [[ -n "${build_ids[${kill_build}]+0}" ]]; then
                      pkill -P "${build_ids[${kill_build}]}"
                   fi
@@ -228,8 +230,8 @@ check_builds()
 
 builds_started=0
 # Now start looping through all of the jobs until everything is done
-while [[ ${builds_started} -lt ${#build_jobs[@]} ]]; do
-   for build in "${!build_jobs[@]}"; do
+while [[ ${builds_started} -lt ${#builds[@]} ]]; do
+   for build in "${!builds[@]}"; do
       # Has the job started?
       if [[ -n "${build_jobs[${build}]+0}" && -z "${build_ids[${build}]+0}" ]]; then
          # Do we have enough processors to run it?
@@ -249,14 +251,14 @@ while [[ ${builds_started} -lt ${#build_jobs[@]} ]]; do
    # Also recalculate how many processors are in use to account for completed builds
    builds_started=0
    procs_in_use=0
-   for build in "${!build_jobs[@]}"; do
+   for build in "${!builds[@]}"; do
       # Has the build started?
       if [[ -n "${build_ids[${build}]+0}" ]]; then
          builds_started=$(( builds_started + 1))
          # Calculate how many processors are in use
          # Is the build still running?
          if ps -p "${build_ids[${build}]}" > /dev/null; then
-            procs_in_use=$(( procs_in_use + build_jobs["${build}"] ))
+            procs_in_use=$(( procs_in_use + builds["${build}"] ))
          fi
       fi
    done
@@ -275,7 +277,7 @@ done
 
 
 # Wait for all jobs to complete and check return statuses
-while [[ "${#build_jobs[@]}" -gt 0 ]]; do
+while [[ "${#builds[@]}" -gt 0 ]]; do
 
    # If requested, check if any build has failed and exit if so
    if [[ "${_quick_kill}" == "YES" ]]; then
@@ -286,7 +288,7 @@ while [[ "${#build_jobs[@]}" -gt 0 ]]; do
       fi
    fi
 
-   for build in "${!build_jobs[@]}"; do
+   for build in "${!builds[@]}"; do
       # Test if each job is complete and if so, notify and remove from the array
       if [[ -n "${build_ids[${build}]+0}" ]]; then
          if ! ps -p "${build_ids[${build}]}" > /dev/null; then
@@ -294,14 +296,14 @@ while [[ "${#build_jobs[@]}" -gt 0 ]]; do
             build_stat=$?
             errs=$((errs+build_stat))
             if [[ ${build_stat} == 0 ]]; then
-               echo "build_${build}.sh completed successfully!"
+               echo "${build_scripts[${build}]} completed successfully!"
             else
-               echo "build_${build}.sh failed with status ${build_stat}!"
+               echo "${build_scripts[${build}]} failed with status ${build_stat}!"
             fi
 
             # Remove the completed build from the list of PIDs
             unset 'build_ids[${build}]'
-            unset 'build_jobs[${build}]'
+            unset 'builds[${build}]'
          fi
       fi
    done

From 30698b8c1c79acf412263a6908d5b383e06d7090 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 19 Dec 2024 13:12:34 +0000
Subject: [PATCH 32/37] Initial WIP stab at compute builds

---
 jobs/JBUILD_COMPILE                          |  16 +++
 jobs/rocoto/build.sh                         |  12 ++
 sorc/build_all.sh                            |  93 ++++++++----
 ush/compute_build.py                         | 143 +++++++++++++++++++
 workflow/applications/application_factory.py |   2 +
 workflow/applications/applications.py        |   4 +-
 workflow/applications/compute_build.py       |  61 ++++++++
 7 files changed, 299 insertions(+), 32 deletions(-)
 create mode 100755 jobs/JBUILD_COMPILE
 create mode 100755 jobs/rocoto/build.sh
 create mode 100755 ush/compute_build.py
 create mode 100644 workflow/applications/compute_build.py

diff --git a/jobs/JBUILD_COMPILE b/jobs/JBUILD_COMPILE
new file mode 100755
index 0000000000..cd0d9ce2fa
--- /dev/null
+++ b/jobs/JBUILD_COMPILE
@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${EXPDIR}/config.base"
+source "${EXPDIR}/config.resources" "${BUILD}"
+
+###############
+# Run the build script
+
+# shellcheck disable=SC2086
+logfile="${HOMEgfs}/sorc/logs/${BUILD_SCRIPT/.sh/}.log"
+"${HOMEgfs}/sorc/${BUILD_SCRIPT}" ${BUILD_FLAGS:-} -j "${ntasks}" 2>&1 | tee "${logfile}"
+
+stat=$?
+
+exit "${stat}"
diff --git a/jobs/rocoto/build.sh b/jobs/rocoto/build.sh
new file mode 100755
index 0000000000..0561554305
--- /dev/null
+++ b/jobs/rocoto/build.sh
@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+export job="compile_${BUILD}"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JBUILD_COMPILE"
+status=$?
+exit "${status}"
diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 89c5fdf0bf..e84102caac 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -15,7 +15,7 @@ function _usage() {
   cat << EOF
 Builds all of the global-workflow components by calling the individual build scripts in parallel.
 
-Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-j n][-v] [gfs] [gefs] [sfs] [gsi] [gdas]
+Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-j n][-v][-K] [gfs] [gefs] [sfs] [gsi] [gdas] [all]
   -a UFS_app:
     Build a specific UFS app instead of the default.  This will be applied to all UFS (GFS, GEFS, SFS) builds.
   -c:
@@ -33,6 +33,8 @@ Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-j n][-v] [gf
   -A:
     HPC account to use for the compute-node builds
     (default is \$HOMEgfs/ci/platforms/config.\$machine:\$HPC_ACCOUNT)
+  -K:
+    Keep temporary files (used for debugging this script)
 
   Specified systems (gfs, gefs, sfs, gsi, gdas) are non-exclusive, so they can be built together.
 EOF
@@ -50,42 +52,41 @@ _build_job_max=20
 _quick_kill="NO"
 _compute_build="NO"
 _hpc_account="default"
+_keep_files="NO"
 # Reset option counter in case this script is sourced
 OPTIND=1
-while getopts ":a:cdfhj:kA:v" option; do
+while getopts ":a:cdfhj:kA:vK" option; do
   case "${option}" in
     a) _build_ufs_opt+="-a ${OPTARG} ";;
     c) _compute_build="YES" ;;
     f) _build_ufs_opt+="-f ";;
     d) _build_debug="-d" ;;
     h) _usage;;
-    k) _quick_kill="YES" ;;
-    A) _hpc_account="${OPTARG}"
-    v) _verbose_opt="-v";;
+    k) _quick_kill="YES" && echo "here2" && exit 2 ;;
+    A) _hpc_account="${OPTARG}" ;;
+    v) _verbose_opt="-v" ;;
+    K) _keep_files="YES" ;;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       _usage
+      exit 1
       ;;
     *)
       echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}"
       _usage
+      exit 1
       ;;
   esac
-  shift $((OPTIND-1))
 done
+shift $((OPTIND-1))
 
-supported_systems=("gfs" "gefs" "sfs" "gsi" "gdas")
-gfs_builds="gfs gfs_utils ufs_utils upp ww3_struct"
+supported_systems=("gfs" "gefs" "sfs" "gsi" "gdas" "all")
+gfs_builds="gfs gfs_utils ufs_utils upp ww3_unstruct"
 gefs_builds="gefs gfs_utils ufs_utils upp ww3_struct"
-sfs_builds="sfs gfs_utils ufs_utils upp ww3_unstruct"
+sfs_builds="sfs gfs_utils ufs_utils upp ww3_struct"
 gsi_builds="gsi_enkf gsi_monitor gsi_utils"
 gdas_builds="gdas gsi_monitor gsi_utils"
-
-# Builds that will use a whole compute node each
-compute_builds="gfs gefs sfs"
-# Service node builds
-# The GDASApp (for now) needs to be compiled on the service node
-service_builds="gdas gfs_utils ufs_utils upp ww3_struct ww3_unstruct gsi gsi_utils gsi_monitor"
+all_builds="gfs gfs_utils ufs_utils upp ww3_unstruct ww3_struct gdas gsi_enkf gsi_monitor gsi_monitor gsi_utils"
 
 # Jobs per build ("min max")
 declare -A build_jobs build_opts build_scripts
@@ -136,7 +137,7 @@ build_scripts["upp"]="build_upp.sh"
 declare -A builds
 system_count=0
 for system in ${@}; do
-   if [[ " ${supported_systems[*]} " =~ " ${system} " ]];
+   if [[ " ${supported_systems[*]} " =~ " ${system} " ]]; then
       (( system_count += 1 ))
       build_list_name="${system}_builds"
       for build in ${!build_list_name}; do
@@ -152,22 +153,11 @@ done
 if [[ ${system_count} -eq 0 ]]; then
    system_count=1
    builds["gfs"]="yes"
+   builds["gfs_utils"]="yes"
+   builds["ufs_utils"]="yes"
+   builds["upp"]="yes"
 fi
 
-# Create directories
-mkdir -p "${HOMEgfs}/sorc/logs" "${HOMEgfs}/exec"
-
-# If we are running this on compute nodes, then call compute_build.sh with the list of builds
-
-if [[ "${_compute_build}" == "YES" ]]; then 
-   #shellcheck disable=SC2086
-   compute_build.sh -a "${_hpc_account}" ${!builds[@]}
-   stat=$?
-   exit ${stat}
-fi
-
-# Otherwise, we are building locally
-
 #------------------------------------
 # GET MACHINE
 #------------------------------------
@@ -179,6 +169,49 @@ if [[ -z "${MACHINE_ID}" ]]; then
   exit 1
 fi
 
+# Create directories
+mkdir -p "${HOMEgfs}/sorc/logs" "${HOMEgfs}/exec"
+
+# If we are running this on compute nodes, then call compute_build.sh with the list of builds
+if [[ "${_compute_build}" == "YES" ]]; then 
+   #####################################################################
+   # COMPUTE NODE BUILD
+   #####################################################################
+   # Load gwsetup module
+   module use "${HOMEgfs}/modulefiles"
+   module load "module_gwsetup.${MACHINE_ID}"
+
+   # Add the workflow to the PYTHONPATH
+   PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/workflow"
+   export PYTHONPATH
+
+   # Prep a build directory
+   build_dir="${HOMEgfs}/sorc/build"
+   rm -rf "${build_dir}"
+   mkdir -p "${build_dir}"
+   cd "${build_dir}"
+
+   # Write the build arrays to a YAML
+   rm -f build_opts.yaml && touch build_opts.yaml
+
+   echo "base:" >> build_opts.yaml
+
+   for build in "${!builds[@]}"; do
+      echo "  BUILD_${build}: YES" >> build_opts.yaml
+      echo "  ${build}_SCRIPT: ${build_scripts[${build}]}" >> build_opts.yaml
+      echo "  ${build}_FLAGS: ${build_opts[${build}]}" >> build_opts.yaml
+   done
+
+   "${HOMEgfs}/ush/compute_build.py" --account "${_hpc_account}" --yaml build_opts.yaml
+   stat=$?
+   if [[ ${stat} == 0 && ${_keep_files:-NO} == "NO" ]]; then
+      rm -rf "${build_dir}"
+   fi
+   exit ${stat}
+fi
+
+# Otherwise, we are building locally, continue in this script
+
 #------------------------------------
 # SOURCE BUILD VERSION FILES
 #------------------------------------
diff --git a/ush/compute_build.py b/ush/compute_build.py
new file mode 100755
index 0000000000..183d988e7d
--- /dev/null
+++ b/ush/compute_build.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+"""
+Entry point for setting up a compute-node build
+"""
+
+import glob
+import os
+import shutil
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+from hosts import Host
+
+import setup_xml
+
+from wxflow import parse_yaml
+
+
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '..'))
+expdir = os.path.join(_top, "sorc", "build")
+
+
+def fill_expdir():
+    """
+    Method to copy config files from workflow to experiment directory
+    """
+    configdir = os.path.join(_top, "parm", "config", "build")
+    configs = glob.glob(f'{configdir}/config.*')
+    if len(configs) == 0:
+        raise IOError(f'no config files found in {configdir}')
+    for config in configs:
+        shutil.copy(config, expdir)
+
+    return
+
+
+def update_configs(host, inputs):
+
+    # Read in the YAML file to fill out templates
+    yaml_path = inputs.yaml
+    if not os.path.exists(yaml_path):
+        raise FileNotFoundError(f'YAML file does not exist, check path: {yaml_path}')
+    yaml_dict = parse_yaml(path=yaml_path)
+
+    # Update config.base
+    base_dict = {
+        "@HOMEgfs@": _top,
+        "@EXPDIR@": expdir,
+        "@MACHINE@": host.machine.upper()}
+
+    # Add/override 'base'-specific declarations in base_dict
+    base_dict = dict(base_dict, **get_template_dict(yaml_dict))
+
+    base_input = os.path.join(_top, "parm", "config", "build", "config.base")
+    base_output = os.path.join(expdir, 'config.base')
+    edit_config(base_input, base_output, host.info, base_dict)
+
+    return
+
+
+def edit_config(input_config, output_config, host_info, config_dict):
+    """
+    Given a templated input_config filename, parse it based on config_dict and
+    host_info and write it out to the output_config filename.
+    """
+
+    # Override defaults with machine-specific defaults
+    host_dict = get_template_dict(host_info)
+    config_dict = dict(config_dict, **host_dict)
+
+    # Read input config
+    with open(input_config, 'rt') as fi:
+        config_str = fi.read()
+
+    # Substitute from config_dict
+    for key, val in config_dict.items():
+        config_str = config_str.replace(key, str(val))
+
+    # Write output config
+    with open(output_config, 'wt') as fo:
+        fo.write(config_str)
+
+    print(f'EDITED:  {output_config} as per user input.')
+
+    return
+
+
+def get_template_dict(input_dict):
+    # Reads a templated input dictionary and updates the output
+
+    output_dict = dict()
+
+    for key, value in input_dict.items():
+        # In some cases, the same config may be templated twice
+        # Prevent adding additional "@"s
+        if "@" in key:
+            output_dict[f'{key}'] = value
+        else:
+            output_dict[f'@{key}@'] = value
+
+    return output_dict
+
+
+def input_args(*argv):
+    """
+    Method to collect user arguments for `setup_build.py`
+    """
+
+    description = """
+        Setup files and directories to start a compute build.
+    """
+
+    parser = ArgumentParser(description=description,
+                            formatter_class=ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('--yaml', help='Input YAML file',
+                        type=str, required=False, default='build_opts.yaml')
+    parser.add_argument('--account', help='HPC account to use; default is host-dependent', required=False, default=os.getenv('HPC_ACCOUNT'))
+
+    inputs = parser.parse_args(list(*argv) if len(argv) else None)
+
+    return inputs
+
+
+def main(*argv):
+
+    user_inputs = input_args(*argv)
+    host = Host()
+
+    # Update the default host account if the user supplied one
+    if user_inputs.account is not None:
+        host.info.ACCOUNT = user_inputs.account
+
+    fill_expdir()
+    update_configs(host, user_inputs)
+
+    setup_xml.main([expdir])
+
+
+if __name__ == '__main__':
+
+    main()
diff --git a/workflow/applications/application_factory.py b/workflow/applications/application_factory.py
index ff6b6992f4..f16858d074 100644
--- a/workflow/applications/application_factory.py
+++ b/workflow/applications/application_factory.py
@@ -2,9 +2,11 @@
 from applications.gfs_cycled import GFSCycledAppConfig
 from applications.gfs_forecast_only import GFSForecastOnlyAppConfig
 from applications.gefs import GEFSAppConfig
+from applications.compute_build import BuildAppConfig
 
 
 app_config_factory = Factory('AppConfig')
 app_config_factory.register('gfs_cycled', GFSCycledAppConfig)
 app_config_factory.register('gfs_forecast-only', GFSForecastOnlyAppConfig)
 app_config_factory.register('gefs_forecast-only', GEFSAppConfig)
+app_config_factory.register('compute_build', BuildAppConfig)
diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py
index 22e299df20..4661bc77d9 100644
--- a/workflow/applications/applications.py
+++ b/workflow/applications/applications.py
@@ -24,7 +24,7 @@ def __call__(cls, *args, **kwargs):
 
 class AppConfig(ABC, metaclass=AppConfigInit):
 
-    VALID_MODES = ['cycled', 'forecast-only']
+    VALID_MODES = ['cycled', 'forecast-only', 'build']
 
     def __init__(self, conf: Configuration) -> None:
 
@@ -59,7 +59,7 @@ def _init_finalize(self, conf: Configuration):
 
         # Now configure the experiment for each valid run
         for run in self.runs:
-            self.configs[run] = self._source_configs(conf, run=run, log=False)
+            self.configs[run] = self._source_configs(conf, run=run, log=True)
 
     def _get_run_options(self, conf: Configuration) -> Dict[str, Any]:
         '''
diff --git a/workflow/applications/compute_build.py b/workflow/applications/compute_build.py
new file mode 100644
index 0000000000..142c039dcc
--- /dev/null
+++ b/workflow/applications/compute_build.py
@@ -0,0 +1,61 @@
+from applications.applications import AppConfig
+from wxflow import Configuration
+from typing import Dict, Any
+
+
+class BuildAppConfig(AppConfig):
+    '''
+    Class to define compute build configurations
+    '''
+    _valid_builds = ["gfs", "gefs", "sfs", "upp", "ufs_utils", "gfs_utils", "gdas", "gsi", "gsi_monitor", "gsi_utils"]
+
+    def __init__(self, conf: Configuration):
+        super().__init__(conf)
+
+        self.runs = ["build"]
+
+    def _get_run_options(self, conf: Configuration) -> Dict[str, Any]:
+
+        # The only build options are the systems to build
+        base = conf.parse_config('config.base')
+        build_options = {}
+        for build in self._valid_builds:
+            build_options[f"build_{build}"] = base.get(f"BUILD_{build}", False)
+
+        run_options = {"build": build_options}
+
+        return run_options
+
+    @staticmethod
+    def _update_base(base_in):
+
+        base_out = base_in.copy()
+        base_out['RUN'] = 'gfs'
+
+        return base_out
+
+    def _get_app_configs(self, run):
+        """
+        Returns the config file required for the build app
+        """
+
+        configs = ['compile']
+
+        return configs
+
+    def get_task_names(self):
+        """
+        Get the task names for all the tasks in the forecast-only application.
+        Note that the order of the task names matters in the XML.
+        This is the place where that order is set.
+        """
+
+        tasks = []
+
+        options = self.run_options["build"]
+
+        for build in self._valid_builds:
+            if options[f'build_{build}']:
+                tasks += [f'compile_{build}']
+
+        return {f"build": tasks}

From 3bdbbd842ec7ae1619e082a34812c3a92a889018 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 19 Dec 2024 14:15:40 +0000
Subject: [PATCH 33/37] Address shellcheck issues

---
 jobs/JBUILD_COMPILE |  3 ++-
 sorc/build_all.sh   | 34 +++++++++++++++++++---------------
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/jobs/JBUILD_COMPILE b/jobs/JBUILD_COMPILE
index cd0d9ce2fa..279a41b7c7 100755
--- a/jobs/JBUILD_COMPILE
+++ b/jobs/JBUILD_COMPILE
@@ -7,8 +7,9 @@ source "${EXPDIR}/config.resources" "${BUILD}"
 ###############
 # Run the build script
 
-# shellcheck disable=SC2086
+# TODO redirect output to sorc/logs/BUILD_SCRIPT.log via rocoto
 logfile="${HOMEgfs}/sorc/logs/${BUILD_SCRIPT/.sh/}.log"
+# shellcheck disable=SC2086,SC2312
 "${HOMEgfs}/sorc/${BUILD_SCRIPT}" ${BUILD_FLAGS:-} -j "${ntasks}" 2>&1 | tee "${logfile}"
 
 stat=$?
diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 810caf177f..cbd348b5d2 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -15,7 +15,7 @@ function _usage() {
   cat << EOF
 Builds all of the global-workflow components by calling the individual build scripts in parallel.
 
-Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-j n][-v][-K] [gfs] [gefs] [sfs] [gsi] [gdas] [all]
+Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-d][-f][-h][-v][-K] [gfs] [gefs] [sfs] [gsi] [gdas] [all]
   -a UFS_app:
     Build a specific UFS app instead of the default.  This will be applied to all UFS (GFS, GEFS, SFS) builds.
   -c:
@@ -67,28 +67,30 @@ while getopts ":a:cdfhj:kA:vK" option; do
     A) _hpc_account="${OPTARG}" ;;
     v) _verbose_opt="-v" ;;
     K) _keep_files="YES" ;;
-    k) _quick_kill="YES" ;;
-    v) _verbose_opt="-v";;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       _usage
-      exit 1
       ;;
     *)
       echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}"
       _usage
-      exit 1
       ;;
   esac
 done
 shift $((OPTIND-1))
 
 supported_systems=("gfs" "gefs" "sfs" "gsi" "gdas" "all")
+# shellcheck disable=SC2034
 gfs_builds="gfs gfs_utils ufs_utils upp ww3_unstruct"
+# shellcheck disable=SC2034
 gefs_builds="gefs gfs_utils ufs_utils upp ww3_struct"
+# shellcheck disable=SC2034
 sfs_builds="sfs gfs_utils ufs_utils upp ww3_struct"
+# shellcheck disable=SC2034
 gsi_builds="gsi_enkf gsi_monitor gsi_utils"
+# shellcheck disable=SC2034
 gdas_builds="gdas gsi_monitor gsi_utils"
+# shellcheck disable=SC2034
 all_builds="gfs gfs_utils ufs_utils upp ww3_unstruct ww3_struct gdas gsi_enkf gsi_monitor gsi_monitor gsi_utils"
 
 # Jobs per build ("min max")
@@ -139,7 +141,8 @@ build_scripts["upp"]="build_upp.sh"
 # Check the requested systems to make sure we can build them
 declare -A builds
 system_count=0
-for system in ${@}; do
+for system in "${@}"; do
+   # shellcheck disable=SC2076
    if [[ " ${supported_systems[*]} " =~ " ${system} " ]]; then
       (( system_count += 1 ))
       build_list_name="${system}_builds"
@@ -191,26 +194,27 @@ if [[ "${_compute_build}" == "YES" ]]; then
    # Prep a build directory
    build_dir="${HOMEgfs}/sorc/build"
    rm -rf "${build_dir}"
-   mkdir -p "${build_dir}"
-   cd "${build_dir}"
 
    # Write the build arrays to a YAML
-   rm -f build_opts.yaml && touch build_opts.yaml
+   build_yaml="${build_dir}/build_opts.yaml"
+   rm -f "${build_yaml}" && touch "${build_yaml}"
 
-   echo "base:" >> build_opts.yaml
+   echo "base:" >> ${build_yaml}
 
    for build in "${!builds[@]}"; do
-      echo "  BUILD_${build}: YES" >> build_opts.yaml
-      echo "  ${build}_SCRIPT: ${build_scripts[${build}]}" >> build_opts.yaml
-      echo "  ${build}_FLAGS: ${build_opts[${build}]}" >> build_opts.yaml
+      {
+         echo "  BUILD_${build}: YES"
+         echo "  ${build}_SCRIPT: ${build_scripts[${build}]}"
+         echo "  ${build}_FLAGS: ${build_opts[${build}]}"
+      } >> "${build_yaml}"
    done
 
-   "${HOMEgfs}/ush/compute_build.py" --account "${_hpc_account}" --yaml build_opts.yaml
+   "${HOMEgfs}/ush/compute_build.py" --account "${_hpc_account}" --yaml ${build_yaml}
    stat=$?
    if [[ ${stat} == 0 && ${_keep_files:-NO} == "NO" ]]; then
       rm -rf "${build_dir}"
    fi
-   exit ${stat}
+   exit "${stat}"
 fi
 
 # Otherwise, we are building locally, continue in this script

From 919bd406979bdb589424e52c49747eceb43315b5 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 19 Dec 2024 14:17:30 +0000
Subject: [PATCH 34/37] Double quote variables

---
 sorc/build_all.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index cbd348b5d2..6cbb91c20b 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -199,7 +199,7 @@ if [[ "${_compute_build}" == "YES" ]]; then
    build_yaml="${build_dir}/build_opts.yaml"
    rm -f "${build_yaml}" && touch "${build_yaml}"
 
-   echo "base:" >> ${build_yaml}
+   echo "base:" >> "${build_yaml}"
 
    for build in "${!builds[@]}"; do
       {
@@ -209,7 +209,7 @@ if [[ "${_compute_build}" == "YES" ]]; then
       } >> "${build_yaml}"
    done
 
-   "${HOMEgfs}/ush/compute_build.py" --account "${_hpc_account}" --yaml ${build_yaml}
+   "${HOMEgfs}/ush/compute_build.py" --account "${_hpc_account}" --yaml "${build_yaml}"
    stat=$?
    if [[ ${stat} == 0 && ${_keep_files:-NO} == "NO" ]]; then
       rm -rf "${build_dir}"

From af37a6c2d9bc210ad1f19572897979c19f4066d4 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 19 Dec 2024 14:33:22 +0000
Subject: [PATCH 35/37] Add build configs

---
 .gitignore                            |  1 -
 parm/config/build/config.base         | 82 +++++++++++++++++++++++++++
 parm/config/build/config.gdas         | 10 ++++
 parm/config/build/config.gefs         | 10 ++++
 parm/config/build/config.gfs          | 10 ++++
 parm/config/build/config.gsi_enkf     | 10 ++++
 parm/config/build/config.gsi_monitor  | 10 ++++
 parm/config/build/config.gsi_utils    | 10 ++++
 parm/config/build/config.resources    | 80 ++++++++++++++++++++++++++
 parm/config/build/config.sfs          | 10 ++++
 parm/config/build/config.ufs_utils    | 10 ++++
 parm/config/build/config.upp          | 10 ++++
 parm/config/build/config.ww3_struct   | 10 ++++
 parm/config/build/config.ww3_unstruct | 10 ++++
 parm/config/build/yaml/defaults.yaml  | 67 ++++++++++++++++++++++
 15 files changed, 339 insertions(+), 1 deletion(-)
 create mode 100644 parm/config/build/config.base
 create mode 100644 parm/config/build/config.gdas
 create mode 100644 parm/config/build/config.gefs
 create mode 100644 parm/config/build/config.gfs
 create mode 100644 parm/config/build/config.gsi_enkf
 create mode 100644 parm/config/build/config.gsi_monitor
 create mode 100644 parm/config/build/config.gsi_utils
 create mode 100644 parm/config/build/config.resources
 create mode 100644 parm/config/build/config.sfs
 create mode 100644 parm/config/build/config.ufs_utils
 create mode 100644 parm/config/build/config.upp
 create mode 100644 parm/config/build/config.ww3_struct
 create mode 100644 parm/config/build/config.ww3_unstruct
 create mode 100644 parm/config/build/yaml/defaults.yaml

diff --git a/.gitignore b/.gitignore
index 49fb3f438a..fe1d0f280d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,7 +18,6 @@ nohup.out
 # Ignore folders
 #-------------------
 exec/
-build*/
 install*/
 
 # Ignore fix directory symlinks
diff --git a/parm/config/build/config.base b/parm/config/build/config.base
new file mode 100644
index 0000000000..a378e0e6e5
--- /dev/null
+++ b/parm/config/build/config.base
@@ -0,0 +1,82 @@
+#! /usr/bin/env bash
+
+########## config.base ##########
+# Common to all steps
+
+echo "BEGIN: config.base"
+
+# Machine environment
+export machine="@MACHINE@"
+
+# Account, queue, etc.
+export ACCOUNT="@ACCOUNT@"
+export QUEUE="@QUEUE@"
+export QUEUE_SERVICE="@QUEUE_SERVICE@"
+export PARTITION_BATCH="@PARTITION_BATCH@"
+export PARTITION_SERVICE="@PARTITION_SERVICE@"
+export RESERVATION="@RESERVATION@"
+export CLUSTERS="@CLUSTERS@"
+export PSLOT="build"
+
+export HOMEgfs="@HOMEgfs@"
+export EXPDIR="@EXPDIR@"
+
+# Dates and assim_freq are required for rocoto
+export SDATE=1900010100
+export EDATE=1900010100
+export assim_freq=6
+
+# Some variables required by wxflow
+export NET=compute
+export MODE=build
+
+# Build options
+export BUILD_gfs=@BUILD_gfs@
+export gfs_SCRIPT=@gfs_SCRIPT@
+export gfs_FLAGS=@gfs_FLAGS@
+
+export BUILD_gefs=@BUILD_gefs@
+export gefs_SCRIPT=@gefs_SCRIPT@
+export gefs_FLAGS=@gefs_FLAGS@
+
+export BUILD_sfs=@BUILD_sfs@
+export sfs_SCRIPT=@sfs_SCRIPT@
+export sfs_FLAGS=@sfs_FLAGS@
+
+export BUILD_gfs_utils=@BUILD_gfs_utils@
+export gfs_utils_SCRIPT=@gfs_utils_SCRIPT@
+export gfs_utils_FLAGS=@gfs_utils_FLAGS@
+
+export BUILD_ufs_utils=@BUILD_ufs_utils@
+export ufs_utils_SCRIPT=@ufs_utils_SCRIPT@
+export ufs_utils_FLAGS=@ufs_utils_FLAGS@
+
+export BUILD_upp=@BUILD_upp@
+export upp_SCRIPT=@upp_SCRIPT@
+export upp_FLAGS=@upp_FLAGS@
+
+export BUILD_ww3_struct=@BUILD_ww3_struct@
+export ww3_struct_SCRIPT=@ww3_struct_SCRIPT@
+export ww3_struct_FLAGS=@ww3_struct_FLAGS@
+
+export BUILD_ww3_unstruct=@BUILD_ww3_unstruct@
+export ww3_unstruct_SCRIPT=@ww3_unstruct_SCRIPT@
+export ww3_unstruct_FLAGS=@ww3_unstruct_FLAGS@
+
+export BUILD_gdas=@BUILD_gdas@
+export gdas_SCRIPT=@gdas_SCRIPT@
+export gdas_FLAGS=@gdas_FLAGS@
+
+export BUILD_gsi_enkf=@BUILD_gsi_enkf@
+export gsi_enkf_SCRIPT=@gsi_enkf_SCRIPT@
+export gsi_enkf_FLAGS=@gsi_enkf_FLAGS@
+
+export BUILD_gsi_utils=@BUILD_gsi_utils@
+export gsi_utils_SCRIPT=@gsi_utils_SCRIPT@
+export gsi_utils_FLAGS=@gsi_utils_FLAGS@
+
+export BUILD_gsi_monitor=@BUILD_gsi_monitor@
+export gsi_monitor_SCRIPT=@gsi_monitor_SCRIPT@
+export gsi_monitor_FLAGS=@gsi_monitor_FLAGS@
+
+echo "END: config.base"
diff --git a/parm/config/build/config.gdas b/parm/config/build/config.gdas
new file mode 100644
index 0000000000..82e47e3b7f
--- /dev/null
+++ b/parm/config/build/config.gdas
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.gdas ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.gdas"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "gdas"
+echo "END: config.gdas"
diff --git a/parm/config/build/config.gefs b/parm/config/build/config.gefs
new file mode 100644
index 0000000000..9942790022
--- /dev/null
+++ b/parm/config/build/config.gefs
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.gefs ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.gefs"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "ufs"
+echo "END: config.gefs"
diff --git a/parm/config/build/config.gfs b/parm/config/build/config.gfs
new file mode 100644
index 0000000000..e9a4e745e1
--- /dev/null
+++ b/parm/config/build/config.gfs
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.gfs ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.gfs"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "ufs"
+echo "END: config.gfs"
diff --git a/parm/config/build/config.gsi_enkf b/parm/config/build/config.gsi_enkf
new file mode 100644
index 0000000000..4a18716117
--- /dev/null
+++ b/parm/config/build/config.gsi_enkf
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.gsi_enkf ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.gsi_enkf"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "gsi_enkf"
+echo "END: config.gsi_enkf"
diff --git a/parm/config/build/config.gsi_monitor b/parm/config/build/config.gsi_monitor
new file mode 100644
index 0000000000..f0a43374c6
--- /dev/null
+++ b/parm/config/build/config.gsi_monitor
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.gsi_monitor ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.gsi_monitor"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "gsi_monitor"
+echo "END: config.gsi_monitor"
diff --git a/parm/config/build/config.gsi_utils b/parm/config/build/config.gsi_utils
new file mode 100644
index 0000000000..a57c345c90
--- /dev/null
+++ b/parm/config/build/config.gsi_utils
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.gsi_utils ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.gsi_utils"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "gsi_utils"
+echo "END: config.gsi_utils"
diff --git a/parm/config/build/config.resources b/parm/config/build/config.resources
new file mode 100644
index 0000000000..67e9ea925d
--- /dev/null
+++ b/parm/config/build/config.resources
@@ -0,0 +1,80 @@
+#! /usr/bin/env bash
+# shellcheck disable=SC2034
+
+########## config.resources ##########
+# Set resource information for job tasks
+# e.g. walltime, node, cores per node, memory etc.
+
+if (( $# != 1 )); then
+
+    echo "Must specify an input task argument to set resource variables!"
+    echo "argument can be any one of the following:"
+    echo "gfs gefs sfs upp ww3_struct ww3_unstruct gdas gsi"
+    echo "gsi_monitor gsi_utils gfs_utils ufs_utils"
+    exit 1
+
+fi
+
+step=$1
+
+echo "BEGIN: config.resources"
+
+case ${step} in
+  "ufs")
+    walltime='00:40:00'
+    ntasks=8
+    ;;
+
+  "gdas")
+    walltime='01:00:00'
+    ntasks=12
+    ;;
+
+  "gsi_enkf")
+    walltime='00:40:00'
+    ntasks=1
+    ;;
+
+  "gsi_monitor")
+    walltime='00:10:00'
+    ntasks=1
+    ;;
+
+  "gsi_utils")
+    walltime='00:10:00'
+    ntasks=1
+    ;;
+
+  "gfs_utils")
+    walltime='00:10:00'
+    ntasks=1
+    ;;
+
+  "upp")
+    walltime='00:15:00'
+    ntasks=1
+    ;;
+
+  "ufs_utils")
+    walltime='00:15:00'
+    ntasks=1
+    ;;
+
+  "ufs_utils")
+    walltime='00:15:00'
+    ntasks=1
+    ;;
+
+  "ww3")
+    walltime='00:15:00'
+    ntasks=1
+    ;;
+
+  *)
+    echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}"
+    exit 1
+    ;;
+
+esac
+
+echo "END: config.resources"
diff --git a/parm/config/build/config.sfs b/parm/config/build/config.sfs
new file mode 100644
index 0000000000..a70e5932d4
--- /dev/null
+++ b/parm/config/build/config.sfs
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.sfs ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.sfs"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "ufs"
+echo "END: config.sfs"
diff --git a/parm/config/build/config.ufs_utils b/parm/config/build/config.ufs_utils
new file mode 100644
index 0000000000..9eba96c5c6
--- /dev/null
+++ b/parm/config/build/config.ufs_utils
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.ufs_utils ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.ufs_utils"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "ufs_utils"
+echo "END: config.ufs_utils"
diff --git a/parm/config/build/config.upp b/parm/config/build/config.upp
new file mode 100644
index 0000000000..0df37b67ca
--- /dev/null
+++ b/parm/config/build/config.upp
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.upp ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.upp"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "upp"
+echo "END: config.upp"
diff --git a/parm/config/build/config.ww3_struct b/parm/config/build/config.ww3_struct
new file mode 100644
index 0000000000..b4135b64b5
--- /dev/null
+++ b/parm/config/build/config.ww3_struct
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.ww3_struct ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.ww3_struct"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "ww3"
+echo "END: config.ww3_struct"
diff --git a/parm/config/build/config.ww3_unstruct b/parm/config/build/config.ww3_unstruct
new file mode 100644
index 0000000000..98e3292c5f
--- /dev/null
+++ b/parm/config/build/config.ww3_unstruct
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+########## config.ww3_unstruct ##########
+# Compilation-specific configuration
+
+echo "BEGIN: config.ww3_unstruct"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" "ww3"
+echo "END: config.ww3_unstruct"
diff --git a/parm/config/build/yaml/defaults.yaml b/parm/config/build/yaml/defaults.yaml
new file mode 100644
index 0000000000..c9ecd11f5b
--- /dev/null
+++ b/parm/config/build/yaml/defaults.yaml
@@ -0,0 +1,67 @@
+base:
+  DOIAU: "YES"
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "NO"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DO_STARTMEM_FROM_JEDIICE: "NO"
+  DO_GOES: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "NO"
+  DO_TRACKER: "YES"
+  DO_GENESIS: "YES"
+  DO_GENESIS_FSU: "NO"
+  DO_METP: "YES"
+  FHMAX_GFS: 120
+  FHMAX_HF_GFS: 0
+  FCST_BREAKPOINTS: ""
+  DO_VRFY_OCEANDA: "NO"
+  GSI_SOILANAL: "NO"
+  EUPD_CYC: "gdas"
+  FHMAX_ENKF_GFS: 12
+  DOHYBVAR_OCN: "NO"
+  DO_TEST_MODE: "NO"
+
+atmanl:
+  JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
+  STATICB_TYPE: "gsibec"
+  LAYOUT_X_ATMANL: 8
+  LAYOUT_Y_ATMANL: 8
+  IO_LAYOUT_X: 1
+  IO_LAYOUT_Y: 1
+
+atmensanl:
+  JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
+  JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2"
+  JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2"
+  LAYOUT_X_ATMENSANL: 8
+  LAYOUT_Y_ATMENSANL: 8
+  IO_LAYOUT_X: 1
+  IO_LAYOUT_Y: 1
+
+aeroanl:
+  IO_LAYOUT_X: 1
+  IO_LAYOUT_Y: 1
+
+snowanl:
+  IO_LAYOUT_X: 1
+  IO_LAYOUT_Y: 1
+
+marineanl:
+  SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
+  SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in oceanprepobs
+  SOCA_NINNER: 100
+
+prepoceanobs:
+  SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
+  SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in ocnanal
+  OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml"
+  use_exp_obs: "YES"
+  dmpdir_exp: "${BASE_DATA}/experimental_obs"
+
+# config.aero has just a system-specific path to add.
+# This is handled by the setup_expt.py, but it has to be told to write to it.
+aero: {}

From d92e1cf982fdffd5d471c9a65a7e6acbbfd23f7c Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 19 Dec 2024 14:35:06 +0000
Subject: [PATCH 36/37] Remove duplicate ufs_utils build resource

---
 parm/config/build/config.resources | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/parm/config/build/config.resources b/parm/config/build/config.resources
index 67e9ea925d..d33559ad54 100644
--- a/parm/config/build/config.resources
+++ b/parm/config/build/config.resources
@@ -60,11 +60,6 @@ case ${step} in
     ntasks=1
     ;;
 
-  "ufs_utils")
-    walltime='00:15:00'
-    ntasks=1
-    ;;
-
   "ww3")
     walltime='00:15:00'
     ntasks=1

From bd93b16291f71ed0bf8d045051ef7c3288fc2711 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 19 Dec 2024 14:37:46 +0000
Subject: [PATCH 37/37] Remove unused build/yaml directory

---
 parm/config/build/yaml/defaults.yaml | 67 ----------------------------
 1 file changed, 67 deletions(-)
 delete mode 100644 parm/config/build/yaml/defaults.yaml

diff --git a/parm/config/build/yaml/defaults.yaml b/parm/config/build/yaml/defaults.yaml
deleted file mode 100644
index c9ecd11f5b..0000000000
--- a/parm/config/build/yaml/defaults.yaml
+++ /dev/null
@@ -1,67 +0,0 @@
-base:
-  DOIAU: "YES"
-  DO_JEDIATMVAR: "NO"
-  DO_JEDIATMENS: "NO"
-  DO_JEDIOCNVAR: "NO"
-  DO_JEDISNOWDA: "NO"
-  DO_MERGENSST: "NO"
-  DO_STARTMEM_FROM_JEDIICE: "NO"
-  DO_GOES: "NO"
-  DO_BUFRSND: "NO"
-  DO_GEMPAK: "NO"
-  DO_AWIPS: "NO"
-  DO_NPOESS: "NO"
-  DO_TRACKER: "YES"
-  DO_GENESIS: "YES"
-  DO_GENESIS_FSU: "NO"
-  DO_METP: "YES"
-  FHMAX_GFS: 120
-  FHMAX_HF_GFS: 0
-  FCST_BREAKPOINTS: ""
-  DO_VRFY_OCEANDA: "NO"
-  GSI_SOILANAL: "NO"
-  EUPD_CYC: "gdas"
-  FHMAX_ENKF_GFS: 12
-  DOHYBVAR_OCN: "NO"
-  DO_TEST_MODE: "NO"
-
-atmanl:
-  JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
-  STATICB_TYPE: "gsibec"
-  LAYOUT_X_ATMANL: 8
-  LAYOUT_Y_ATMANL: 8
-  IO_LAYOUT_X: 1
-  IO_LAYOUT_Y: 1
-
-atmensanl:
-  JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
-  JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2"
-  JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2"
-  LAYOUT_X_ATMENSANL: 8
-  LAYOUT_Y_ATMENSANL: 8
-  IO_LAYOUT_X: 1
-  IO_LAYOUT_Y: 1
-
-aeroanl:
-  IO_LAYOUT_X: 1
-  IO_LAYOUT_Y: 1
-
-snowanl:
-  IO_LAYOUT_X: 1
-  IO_LAYOUT_Y: 1
-
-marineanl:
-  SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
-  SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in oceanprepobs
-  SOCA_NINNER: 100
-
-prepoceanobs:
-  SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
-  SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in ocnanal
-  OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml"
-  use_exp_obs: "YES"
-  dmpdir_exp: "${BASE_DATA}/experimental_obs"
-
-# config.aero has just a system-specific path to add.
-# This is handled by the setup_expt.py, but it has to be told to write to it.
-aero: {}