forked from NOAA-EMC/global-workflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
50 changed files
with
936 additions
and
669 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| if [[ $# -ne 1 ]]; then | ||
|
|
||
| echo "Must specify an input argument to set runtime environment variables!" | ||
| exit 1 | ||
|
|
||
| fi | ||
|
|
||
| step=$1 | ||
|
|
||
| export launcher="srun -l --export=ALL" | ||
| export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" | ||
|
|
||
| # Configure MPI environment | ||
| export OMP_STACKSIZE=2048000 | ||
| export NTHSTACK=1024000000 | ||
|
|
||
| ulimit -s unlimited | ||
| ulimit -a | ||
|
|
||
| # Calculate common variables | ||
| # Check first if the dependent variables are set | ||
| if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then | ||
| max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) | ||
| NTHREADSmax=${threads_per_task:-${max_threads_per_task}} | ||
| NTHREADS1=${threads_per_task:-1} | ||
| [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} | ||
| [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} | ||
| export APRUN="${launcher} -n ${ntasks}" | ||
| else | ||
| echo "ERROR config.resources must be sourced before sourcing AZUREPW.env" | ||
| exit 2 | ||
| fi | ||
|
|
||
| if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then | ||
|
|
||
| export launcher="srun --mpi=pmi2 -l" | ||
|
|
||
| (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) | ||
| (( ufs_ntasks = nnodes*tasks_per_node )) | ||
| # With ESMF threading, the model wants to use the full node | ||
| export APRUN_UFS="${launcher} -n ${ufs_ntasks}" | ||
| unset nnodes ufs_ntasks | ||
|
|
||
| elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then | ||
|
|
||
| export CFP_MP="YES" | ||
| if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi | ||
| export wavempexec=${launcher} | ||
| export wave_mpmd=${mpmd_opt} | ||
|
|
||
| elif [[ "${step}" = "post" ]]; then | ||
|
|
||
| export NTHREADS_NP=${NTHREADS1} | ||
| export APRUN_NP="${APRUN}" | ||
|
|
||
| export NTHREADS_DWN=${threads_per_task_dwn:-1} | ||
| [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} | ||
| export APRUN_DWN="${launcher} -n ${ntasks_dwn}" | ||
|
|
||
| elif [[ "${step}" = "atmos_products" ]]; then | ||
|
|
||
| export USE_CFP="YES" # Use MPMD for downstream product generation on Hera | ||
|
|
||
| elif [[ "${step}" = "oceanice_products" ]]; then | ||
|
|
||
| export NTHREADS_OCNICEPOST=${NTHREADS1} | ||
| export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" | ||
|
|
||
| elif [[ "${step}" = "ecen" ]]; then | ||
|
|
||
| export NTHREADS_ECEN=${NTHREADSmax} | ||
| export APRUN_ECEN="${APRUN}" | ||
|
|
||
| export NTHREADS_CHGRES=${threads_per_task_chgres:-12} | ||
| [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} | ||
| export APRUN_CHGRES="time" | ||
|
|
||
| export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} | ||
| [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} | ||
| export APRUN_CALCINC="${APRUN}" | ||
|
|
||
| elif [[ "${step}" = "esfc" ]]; then | ||
|
|
||
| export NTHREADS_ESFC=${NTHREADSmax} | ||
| export APRUN_ESFC="${APRUN}" | ||
|
|
||
| export NTHREADS_CYCLE=${threads_per_task_cycle:-14} | ||
| [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} | ||
| export APRUN_CYCLE="${APRUN}" | ||
|
|
||
| elif [[ "${step}" = "epos" ]]; then | ||
|
|
||
| export NTHREADS_EPOS=${NTHREADSmax} | ||
| export APRUN_EPOS="${APRUN}" | ||
|
|
||
| elif [[ "${step}" = "fit2obs" ]]; then | ||
|
|
||
| export NTHREADS_FIT2OBS=${NTHREADS1} | ||
| export MPIRUN="${APRUN}" | ||
|
|
||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| # AZURE-specific job resources | ||
|
|
||
| export is_exclusive="True" | ||
| unset memory | ||
|
|
||
| # shellcheck disable=SC2312 | ||
| for mem_var in $(env | grep '^memory_' | cut -d= -f1); do | ||
| unset "${mem_var}" | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.