merge upstream develop; resolve conflicts

.github/scripts/check_tech_doc.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# This script recreates technical documentation for the ush and tests/WE2E Python scripts
+# If the tech docs produced here do not match the branch's contents, the script will fail
+
+set -eo pipefail
+
+# Install prerequisites
+pip install sphinx
+pip install sphinx-rtd-theme
+pip install sphinxcontrib-bibtex
+
+# Regenerate tech docs in ush and tests/WE2E based on current state of scripts in those directories.
+cd doc/TechDocs
+sphinx-apidoc -fM --remove-old -o ./ush ../../ush
+sphinx-apidoc -fM --remove-old -o ./tests/WE2E ../../tests/WE2E
+
+# Check for mismatch between what comes out of this action and what is in the PR. 
+status=`git status -s`
+
+if [ -n "${status}" ]; then
+  echo ${status}
+  echo ""
+  echo "Please update your Technical Documentation RST files."
+  exit 1
+else
+  echo "Technical documentation is up-to-date."
+  exit 0
+fi

.github/workflows/doc_tests.yaml

@@ -0,0 +1,25 @@
+name: Doc Tests
+on:
+  push:
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+jobs:
+  doc_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Check tech docs
+        run: .github/scripts/check_tech_doc.sh
+      - name: Build documentation
+        run: |
+          cd doc
+          make doc

.github/workflows/python_tests.yaml

@@ -30,10 +30,6 @@ jobs:
           cache-downloads: true
           cache-environment: true
 
-      - name: Checkout externals
-        run: |
-          ./manage_externals/checkout_externals ufs-weather-model
-
       - name: Lint the python code
         run: |
           micromamba activate srw_app
@@ -44,17 +40,22 @@ jobs:
           pylint ush/set_fv3nml*.py
           pylint ush/update_input_nml.py
 
+      - name: Checkout externals
+        run: |
+          ./manage_externals/checkout_externals ufs-weather-model
+
       - name: Run python unittests
         run: |
           # exclude test_retrieve_data that is tested in functional test
           micromamba activate srw_app
           export UNIT_TEST=True
           export PYTHONPATH=$(pwd)/ush
-          python -m unittest -b tests/test_python/*.py
+          python -m unittest tests/test_python/*.py
 
       - name: Run python functional tests
         run: |
           micromamba activate srw_app
           export CI=true
           export PYTHONPATH=${PWD}/ush
-          python3 -m unittest -b tests/test_python/test_retrieve_data.py
+          python3 -m unittest tests/test_python/test_retrieve_data.py
+

CMakeLists.txt

@@ -8,12 +8,20 @@ project(ufs-srweather-app VERSION 1.0 LANGUAGES C CXX Fortran)
 find_package(MPI REQUIRED COMPONENTS C CXX Fortran)
 
 # Set extended version info.
-SET(SRWA_VERSION_MAJOR 1)
-SET(SRWA_VERSION_MINOR 0)
+SET(SRWA_VERSION_MAJOR 2)
+SET(SRWA_VERSION_MINOR 2)
 SET(SRWA_VERSION_PATCH 0)
 SET(SRWA_VERSION_NOTE "-development")
 SET(SRWA_VERSION ${SRWA_VERSION_MAJOR}.${SRWA_VERSION_MINOR}.${SRWA_VERSION_PATCH}${SRWA_VERSION_NOTE})
 
+# Get the latest abbreviated commit hash of the working branch
+execute_process(
+    COMMAND git log -1 --format=%h
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+    OUTPUT_VARIABLE GIT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
 # A function used to create autotools-style 'yes/no' definitions.
 # If a variable is set, it 'yes' is returned. Otherwise, 'no' is
 # returned.
@@ -55,7 +63,7 @@ if(NOT DEFINED CMAKE_INSTALL_BINDIR)
 endif()
 
 #####
-# Configure and print the ufs-srweather-app.settings file.
+# Configure and print the build settings file.
 #####
 
 # Determine the configure date.
@@ -92,10 +100,12 @@ SET(host_vendor "${osname}")
 SET(host_os "${osrel}")
 SET(abs_top_builddir "${CMAKE_CURRENT_BINARY_DIR}")
 SET(abs_top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}")
+SET(application "${APP}")
+SET(machine "${BUILD_MACHINE}")
 
 SET(CC_VERSION "${CMAKE_C_COMPILER}")
 
-# Set values for .settings file.
+# Set values for build settings file.
 SET(CFLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}}")
 SET(CPPFLAGS "${CMAKE_CPP_FLAGS} ${CMAKE_CPP_FLAGS_${CMAKE_BUILD_TYPE}}")
 SET(LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS_${CMAKE_BUILD_TYPE}}")
@@ -107,27 +117,27 @@ is_enabled(BUILD_SHARED_LIBS enable_shared)
 is_enabled(STATUS_PARALLEL HAS_PARALLEL)
 
 # Generate file from template.
-CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/ufs_srweather_app.settings.in"
-  "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app.settings"
+CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/sorc/build_settings_template.yaml"
+  "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml"
   @ONLY)
 
 # Read in settings file, print out.
 # Avoid using system-specific calls so that this
 # might also work on Windows.
-FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app.settings"
+FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml"
   UFS-SRWEATHER-APP_SETTINGS)
 MESSAGE(${UFS-SRWEATHER-APP_SETTINGS})
 
-# Install ufs_srweather_app.settings file into same location
+# Install build settings file into same location
 # as the app.
-INSTALL(FILES "${CMAKE_BINARY_DIR}/ufs_srweather_app.settings"
+INSTALL(FILES "${CMAKE_BINARY_DIR}/build_settings.yaml"
   DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 #####
 # Create 'ufs_srweather_app_meta.h' include file.
 #####
 configure_file(
-  ufs_srweather_app_meta.h.in
+  sorc/ufs_srweather_app_meta.h.in
   ufs_srweather_app_meta.h @ONLY)
 
 FILE(COPY "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app_meta.h" DESTINATION include)

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 38a29a6
+hash = 6b0f516
 local_path = sorc/ufs-weather-model
 required = True
 
@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 81b38a8
+hash = 6f5dd62
 local_path = sorc/UPP
 required = True
 

devbuild.sh

@@ -15,8 +15,10 @@ OPTIONS
       compiler to use; default depends on platform
       (e.g. intel | gnu | cray | gccgfortran)
   -a, --app=APPLICATION
-      weather model application to build; for example, ATMAQ for Online-CMAQ
-      (e.g. ATM | ATMAQ | ATMW | S2S | S2SW)
+      weather model application to build; supported SRW options are
+        ATM    (default) Atmosphere only
+        ATMAQ  Online-CMAQ (air quality)
+        ATMF   UFS_FIRE (coupled Community Fire Behavior Model)
   --ccpp="CCPP_SUITE1,CCPP_SUITE2..."
       CCPP suites (CCPP_SUITES) to include in build; delimited with ','
   --enable-options="OPTION1,OPTION2,..."
@@ -363,6 +365,7 @@ fi
 
 # cmake settings
 CMAKE_SETTINGS="\
+ -DBUILD_MACHINE=${MACHINE}\
  -DCMAKE_BUILD_TYPE=${BUILD_TYPE}\
  -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}\
  -DCMAKE_INSTALL_BINDIR=${BIN_DIR}\

doc/ContribGuide/documentation.rst

@@ -69,4 +69,54 @@ Please follow these guidelines when contributing to the documentation:
 
         .. code-block:: python
 
-           n = 88
+           n = 88
+
+Troubleshooting Documentation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the SRW App documentation Makefile (``ufs-srweather-app/doc/Makefile``), the ``-W`` option causes documentation builds to fail when there are errors or warnings in the build. 
+This ensures that the documentation remains up-to-date and notifies developers of any new issues (like failing links). However, the build will fail when it hits the first error without showing subsequent errors. 
+When troubleshooting, it can be useful to see all warnings and errors. 
+To see all warnings and errors, comment out the ``-W`` flag in ``SPHINXOPTS`` in the Makefile and build the documentation by running ``make doc`` from the ``doc`` directory. 
+
+Technical Documentation Guidelines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Technical (API-like) documentation is generated for any Python scripts in the ``ush`` or ``tests/WE2E`` directories. 
+When developers change Python files in these directories, they need to update the Python docstrings (i.e., comments in ``"""triple quotes"""``) to reflect the changes they made. 
+Each Python script should include: 
+
+   * A summary of the script's purpose/functionality
+
+      * Should start with a verb, e.g., "checks" or "loads" or "initializes"
+
+   * Docstrings for each method (except methods like ``_parse_args`` that start with an underscore). These docstrings should include:
+
+      * A description of what the method does (starting with a verb, e.g., "checks" or "parses")
+      * A list of method parameters, or ``Args:``, with a definition and expected data type for each
+      * A return statement (``Returns:``) -- if applicable
+      * List of exceptions (``Raises:``) -- if applicable
+
+.. note:: 
+
+   Python does not have truly private methods, but methods that start with an underscore are the closest equivalent. In the SRW App, the underscore signals that this method is only accessed by the script within which it is called. 
+
+After updating the docstrings, developers need to update the corresponding RST files. 
+To do this successfully, developers must have *sphinx>=7.4.0* installed on their system. To update the RST files, run: 
+
+.. code-block:: console
+
+   cd ufs-srweather-app/doc/TechDoc
+   sphinx-apidoc -fM --remove-old -o ./ush ../../ush
+   sphinx-apidoc -fM --remove-old -o ./tests/WE2E ../../tests/WE2E
+
+.. note:: 
+
+   Developers who do not have *sphinx>=7.4.0* installed may issue the following commands from ``ufs-srweather-app/doc/TechDoc`` before running the sphinx-apidoc commands above: 
+
+   .. code-block:: console
+
+      rm -rf ush
+      rm -rf tests/WE2E
+   
+   This will delete current RST files before recreating them with the ``sphinx-apidoc`` command based on the current contents of ``ush`` and ``tests/WE2E``. This step is necessary because the ``--remove-old`` flag does not work with earlier versions of sphinx. 

doc/Makefile

@@ -24,4 +24,4 @@ linkcheck:
 # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
 
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log

doc/TechDocs/ush/eval_metplus_timestr_tmpl.rst

@@ -0,0 +1,7 @@
+eval\_metplus\_timestr\_tmpl module
+===================================
+
+.. automodule:: eval_metplus_timestr_tmpl
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/get_obs.rst

@@ -0,0 +1,7 @@
+get\_obs module
+===============
+
+.. automodule:: get_obs
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/modules.rst

@@ -12,18 +12,21 @@ ush
    create_diag_table_file
    create_model_configure_file
    create_ufs_configure_file
+   eval_metplus_timestr_tmpl
    generate_FV3LAM_wflow
    get_crontab_contents
+   get_obs
    link_fix
    mrms_pull_topofhour
    python_utils
    retrieve_data
    run_srw_tests
-   set_cycle_dates
+   set_cycle_and_obs_timeinfo
    set_fv3nml_ens_stoch_seeds
    set_fv3nml_sfc_climo_filenames
    set_gridparams_ESGgrid
    set_gridparams_GFDLgrid
+   set_leadhrs
    set_predef_grid_params
    setup
    update_input_nml

doc/TechDocs/ush/set_cycle_and_obs_timeinfo.rst

@@ -0,0 +1,7 @@
+set\_cycle\_and\_obs\_timeinfo module
+=====================================
+
+.. automodule:: set_cycle_and_obs_timeinfo
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/set_leadhrs.rst

@@ -0,0 +1,7 @@
+set\_leadhrs module
+===================
+
+.. automodule:: set_leadhrs
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/UsersGuide/BuildingRunningTesting/DefaultVarsTable.rst

@@ -20,7 +20,7 @@ Table of Variables in ``config_defaults.yaml``
        TEST_VX_FCST_INPUT_BASEDIR, FIXgsm, FIXaer, FIXlut, FIXorg, FIXsfc, FIXshp, FIXcrtm, FIXcrtmupp, EXTRN_MDL_DATA_STORES
    * - Workflow
      - WORKFLOW_ID, RELATIVE_LINK_FLAG, USE_CRON_TO_RELAUNCH, CRON_RELAUNCH_INTVL_MNTS, CRONTAB_LINE, LOAD_MODULES_RUN_TASK_FP, EXPT_BASEDIR, EXPT_SUBDIR, EXEC_SUBDIR, 
-       EXPTDIR, DOT_OR_USCORE, EXPT_CONFIG_FN, CONSTANTS_FN, RGNL_GRID_NML_FN, FV3_NML_FN, FV3_NML_BASE_SUITE_FN, FV3_NML_YAML_CONFIG_FN, FV3_NML_BASE_ENS_FN, 
+       EXPTDIR, DOT_OR_USCORE, CONSTANTS_FN, RGNL_GRID_NML_FN, FV3_NML_FN, FV3_NML_BASE_SUITE_FN, FV3_NML_YAML_CONFIG_FN, FV3_NML_BASE_ENS_FN, 
        FV3_EXEC_FN, DATA_TABLE_FN, DIAG_TABLE_FN, FIELD_TABLE_FN, DIAG_TABLE_TMPL_FN, FIELD_TABLE_TMPL_FN, MODEL_CONFIG_FN, NEMS_CONFIG_FN, AQM_RC_FN, AQM_RC_TMPL_FN, 
        FV3_NML_BASE_SUITE_FP, FV3_NML_YAML_CONFIG_FP, FV3_NML_BASE_ENS_FP, DATA_TABLE_TMPL_FP, DIAG_TABLE_TMPL_FP, FIELD_TABLE_TMPL_FP, 
        MODEL_CONFIG_TMPL_FP, NEMS_CONFIG_TMPL_FP, AQM_RC_TMPL_FP, DATA_TABLE_FP, FIELD_TABLE_FP, NEMS_CONFIG_FP, FV3_NML_FP,