Skip to content

Commit

Permalink
Merge branch 'NOAA-EMC:develop' into feature/GW-RT_GlobalNML
Browse files Browse the repository at this point in the history
  • Loading branch information
dpsarmie authored Oct 29, 2024
2 parents f379c15 + b56f633 commit 51ecf63
Show file tree
Hide file tree
Showing 48 changed files with 861 additions and 215 deletions.
3 changes: 3 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -208,3 +208,6 @@ ush/python/pygfs/ufswm/gfs.py @aerorahul
ush/python/pygfs/ufswm/ufs.py @aerorahul
ush/python/pygfs/utils/__init__.py @aerorahul
ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA

# Specific workflow scripts
workflow/generate_workflows.sh @DavidHuber-NOAA
1 change: 1 addition & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,5 @@ Example:
- [ ] My changes generate no new warnings
- [ ] New and existing tests pass with my changes
- [ ] This change is covered by an existing CI test or a new one has been added
- [ ] Any new scripts have been added to the .github/CODEOWNERS file with owners
- [ ] I have made corresponding changes to the system documentation if necessary
42 changes: 23 additions & 19 deletions ci/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ def cases = ''
def GH = 'none'
// Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR.
def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
def repo_url = '[email protected]:NOAA-EMC/global-workflow.git'
def STATUS = 'Passed'

Expand Down Expand Up @@ -101,9 +101,10 @@ pipeline {
stages {
stage('Building') {
steps {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
script {
def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
sh(script: "mkdir -p ${HOMEgfs}")
ws(HOMEgfs) {
if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
Expand Down Expand Up @@ -172,9 +173,10 @@ pipeline {
}
if (system == 'gfs') {
cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
echo "Cases to run: ${cases}"
}
}
}
}
}
}
}
Expand All @@ -192,32 +194,34 @@ pipeline {
def parallelStages = cases.collectEntries { caseName ->
["${caseName}": {
stage("Create ${caseName}") {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
script {
sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
system = yaml_case.experiment.system
def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis
env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
try {
error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
} catch (Exception error_create) {
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
error("Case ${caseName} failed to create experment directory")
}
script {
sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
def build_system = yaml_case.experiment.system
def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}" // local HOMEgfs is used to populate the XML on per system basis
env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
try {
error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
} catch (Exception error_create) {
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
error("Case ${caseName} failed to create experment directory")
}
}
}
}

stage("Running ${caseName}") {
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
script {
HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
sh(script: " rm -f ${error_file}")
def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
def build_system = yaml_case.experiment.system
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
} catch (Exception error_experment) {
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
ws(CUSTOM_WORKSPACE) {
Expand Down Expand Up @@ -268,11 +272,11 @@ pipeline {
}
}


stage( '5. FINALIZE' ) {
agent { label NodeName[machine].toLowerCase() }
steps {
script {
env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
sh(script: """
labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
for label in \$labels; do
Expand Down
2 changes: 1 addition & 1 deletion ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ arguments:
resdetatmos: 384
resdetocean: 0.25
nens: 0
gfs_cyc: 4
interval: 6
start: cold
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
Expand Down
2 changes: 1 addition & 1 deletion ci/cases/pr/C48_S2SWA_gefs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ arguments:
resdetocean: 5.0
resensatmos: 48
nens: 2
gfs_cyc: 1
interval: 24
start: cold
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
Expand Down
2 changes: 1 addition & 1 deletion ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ arguments:
idate: 2021032412
edate: 2021032418
nens: 0
gfs_cyc: 0
interval: 0
start: warm
yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml

Expand Down
2 changes: 1 addition & 1 deletion ci/cases/pr/C96C48_hybatmDA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ arguments:
idate: 2021122018
edate: 2021122106
nens: 2
gfs_cyc: 1
interval: 24
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
3 changes: 2 additions & 1 deletion ci/cases/pr/C96C48_hybatmaerosnowDA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@ arguments:
idate: 2021122012
edate: 2021122100
nens: 2
gfs_cyc: 1
interval: 24
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

skip_ci_on_hosts:
- wcoss2
- orion
- gaea
- hercules
2 changes: 1 addition & 1 deletion ci/cases/pr/C96C48_ufs_hybatmDA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ arguments:
idate: 2024022318
edate: 2024022406
nens: 2
gfs_cyc: 1
interval: 24
start: warm
yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

Expand Down
5 changes: 4 additions & 1 deletion ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@ arguments:
resdetocean: 1.0
resensatmos: 96
nens: 2
gfs_cyc: 1
interval: 6
start: warm
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
idate: 2020110100
edate: 2020110100
yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml
icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610

skip_ci_on_hosts:
- wcoss2
2 changes: 1 addition & 1 deletion ci/cases/pr/C96_atm3DVar.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ arguments:
idate: 2021122018
edate: 2021122106
nens: 0
gfs_cyc: 1
interval: 24
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

Expand Down
2 changes: 1 addition & 1 deletion ci/cases/pr/C96_atm3DVar_extended.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ arguments:
idate: 2021122018
edate: 2021122118
nens: 0
gfs_cyc: 4
interval: 6
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml

Expand Down
2 changes: 1 addition & 1 deletion ci/cases/weekly/C384C192_hybatmda.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ arguments:
idate: 2023040118
edate: 2023040200
nens: 2
gfs_cyc: 1
interval: 24
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
2 changes: 1 addition & 1 deletion ci/cases/weekly/C384_atm3DVar.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ arguments:
idate: 2023040118
edate: 2023040200
nens: 0
gfs_cyc: 1
interval: 24
start: cold
yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
2 changes: 1 addition & 1 deletion ci/scripts/utils/launch_java_agent.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh"
[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
${GH} --version

check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
if [[ "${check_mark}" != "" ]]; then
echo "gh not authenticating with emcbot token"
exit 1
Expand Down
2 changes: 1 addition & 1 deletion docs/source/clone.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Quick Instructions
Quick clone/build/link instructions (more detailed instructions below).

.. note::
Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.
Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.

Clone the `global-workflow` and `cd` into the `sorc` directory:

Expand Down
25 changes: 24 additions & 1 deletion docs/source/development.rst
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Code managers

* Kate Friedman - @KateFriedman-NOAA / [email protected]
* Walter Kolczynski - @WalterKolczynski-NOAA / [email protected]
* David Huber - @DavidHuber-NOAA / [email protected]

.. _development:

Expand Down Expand Up @@ -70,7 +71,29 @@ The following steps should be followed in order to make changes to the develop b
Development Tools
=================

See the ``/test`` folder in global-workflow for available development and testing tools.
Two sets of testing are available for use by developers. The first is the capability to run continuous integration tests locally and the second are a set of comparison tools.

---------------------------
Continuous Integration (CI)
---------------------------

The global workflow comes fitted with a suite of system tests that run various types of workflow. These tests are commonly run for pull requests before they may be merged into the develop branch. At a minimum, developers are expected to run the CI test(s) that will be impacted by their changes on at least one platform.

The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory. The ``workflow/generate_workflows.sh`` tool is available to aid running these cases. See the help documentation by running ``./generate_workflows.sh -h``. The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :doc:`setup` for details on these directories). The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS). For instance, to build the workflow and run all of the GFS tests, one would execute

::

cd workflow
./generate_workflows.sh -A "your_hpc_account" -b -G -c /path/to/root/directory

where:

* ``-A`` is used to specify the HPC (slurm or PBS) account to use
* ``-b`` indicates that the workflow should be built fresh
* ``-G`` specifies that all of the GFS cases should be run (this also influences the build flags to use)
* ``-c`` tells the tool to append the rocotorun commands for each experiment to your crontab

More details on how to use the tool are provided by running ``generate_workflows.sh -h``.

----------------
Comparison Tools
Expand Down
4 changes: 2 additions & 2 deletions docs/source/hpc.rst
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ Optimizing the global workflow on S4

The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui ([email protected]) if you are planning on running a cycled experiment on this system to obtain optimized configuration files.

========================================
==================================================
Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4)
========================================
==================================================

Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components:

Expand Down
2 changes: 1 addition & 1 deletion docs/source/init.rst
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh
| | | | |
| | gfs.t. ``hh`` z.sfcanl.nc | | |
+----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+
| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos|
| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs\_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos|
| | | | |
| | gfs.t. ``hh`` z.sfcanl.nc | | |
+----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+
Expand Down
2 changes: 1 addition & 1 deletion docs/source/jobs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ GFS Configuration

The sequence of jobs that are run for an end-to-end (analysis+forecast+post processing+verification) GFS configuration using the Global Workflow is shown above. The system utilizes a collection of scripts that perform the tasks for each step.

For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. As with the operational system, the gdas runs for each cycle (00, 06, 12, and 18 UTC), however, to save time and space in experiments, the gfs (right side of the diagram) is initially setup to run for only the 00 UTC cycle (See the "run GFS this cycle?" portion of the diagram). The option to run the GFS for all four cycles is available (see the ``gfs_cyc`` variable in configuration file).
For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system.

An experimental run is different from operations in the following ways:

Expand Down
Loading

0 comments on commit 51ecf63

Please sign in to comment.