#162 Merge branch 'main' of github.com:NOAA-GFDL/fre-cli into 162.fi…

…x-yaml-overwrites

.gitignore

@@ -1,3 +1,8 @@
+# fremake generated files that should be ignored
+combined-null_model.yaml
+combined-am5.yaml
+tmp
+makefile_out
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

docs/setup.rst

@@ -1,14 +1,14 @@
 =====
 Setup
 =====
-fre-cli is conda-installable from the “noaa-gfdl” anaconda channel (https://anaconda.org/NOAA-GFDL/fre-cli)
+FRE-cli is conda-installable from the “noaa-gfdl” anaconda channel (https://anaconda.org/NOAA-GFDL/fre-cli)
 and is deployed on GFDL systems as Environment Modules.
 
 On GFDL systems
 ========================
 If you are at GFDL (gaea, PP/AN, workstations), you may skip installation::
 
-  module load fre/2024.01
+  module load fre/2025.01
 
   fre --help
 
@@ -26,7 +26,7 @@ Once you have conda available, install the latest fre-cli from the NOAA-GFDL ana
 
 To install a specific version::
 
-  conda create --name fre-202401 --channel noaa-gfdl --channel conda-forge fre-cli::2024.01
+  conda create --name fre-202501 --channel noaa-gfdl --channel conda-forge fre-cli::2025.01
 
 and activate it::
 

docs/tools/cmor.rst

@@ -75,7 +75,7 @@ background
 ~~~~~~~~~~
 
 The bulk of this routine is housed in ``fre/cmor/cmor_mixer.py``, which is a rewritten version of
-Sergey Malyshev's original ``CMORcommander.py`` script, utilized during GFDL's CMIP6 publishing run.
+Sergey Nikonov's original ``CMORcommander.py`` script, utilized during GFDL's CMIP6 publishing run.
 
 This code is dependent on two primary json configuration files- a MIP
 variable table and another containing experiment (i.e. model) specific metdata (e.g. grid) to append

docs/tools/yamltools.rst

@@ -1,6 +1,13 @@
-.. NEEDS UPDATING #TODO
-   
 ``combine-yamls``
 -----------------
 
-* placehold
+``fre yamltools combine-yamls [options]``
+   - Purpose: Creates a combined yaml file for either compilation or post-processing. 
+              If `--use compile`, the model yaml is combined with the compile and platforms yaml.
+              If `--use pp`, the model yaml is combined with post-processing yamls.
+   - Options:
+        - `-y, --yamlfile [experiment yaml] (required)`
+        - `-e, --experiment [experiment name]`
+        - `-p, --platform [platform] (required)`
+        - `-t, --target [target] (required)`
+        - `--use [compile|pp] (required)`

docs/usage.rst

@@ -1,7 +1,7 @@
 =============
 Usage
 =============
-Using a set of YAML configuration files, ``fre make`` compiles a FMS-based model, and ``fre pp`` postprocesses the history output and runs diagnostic analysis scripts. Please note that model running is not yet supported in FRE 2024; continue to use FRE Bronx frerun.
+Using a set of YAML configuration files, ``fre make`` compiles a FMS-based model, and ``fre pp`` postprocesses the history output and runs diagnostic analysis scripts. Please note that model running is not yet supported in FRE 2025; continue to use FRE Bronx frerun.
 
 YAML Framework
 ========================

docs/usage/compile.rst

@@ -82,7 +82,7 @@ Guide
 
 For the container build, parallel checkouts are not supported, so the `-npc` options must be used for the checkout script. In addition the platform must be a container platform.
 
-Users will not be able to create containers unless they have podman access on gaea.
+Gaea users will not be able to create containers unless they have requested and been given podman access.
 
 .. code-block::
 
@@ -103,45 +103,45 @@ Users will not be able to create containers unless they have podman access on ga
 
 Quickstart
 ----------
-The quickstart instructions can be used with the am5-compile examples located in the fre-examples repository: https://github.com/NOAA-GFDL/fre-examples/tree/main/AM5/am5-compile
+The quickstart instructions can be used with the null model example located in the fre-cli repository: https://github.com/NOAA-GFDL/fre-cli/tree/main/fre/make/tests/null_example
 
 1. Bare-metal Build:
 
 .. code-block::
 
   # Create checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod
 
   # Create and run checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod --execute
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod --execute
 
   # Create Makefile
-  fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod
+  fre make create-makefile -y null_model.yaml -p ncrc5.intel23 -t prod
 
   # Create the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod
 
   # Create and run the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod --execute
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod --execute
 
 2. Bare-metal Build Multi-target:
 
 .. code-block::
 
   # Create checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod -t debug
 
   # Create and run checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod -t debug --execute
 
   # Create Makefile
-  fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+  fre make create-makefile -y null_model.yaml -p ncrc5.intel23 -t prod -t debug
 
   # Create the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod -t debug
 
   # Create and run the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod -t debug --execute
 
 3. Container Build:
 
@@ -150,28 +150,34 @@ In order for the container to build successfully, a `-npc`, or `--no-parallel-ch
 .. code-block::
 
   # Create checkout script
-  fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc
+  fre make create-checkout -y null_model.yaml -p hpcme.2023 -t prod -npc
 
   # Create and run checkout script
-  fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc --execute
+  fre make create-checkout -y null_model.yaml -p hpcme.2023 -t prod -npc --execute
 
   # Create Makefile
-  fre make create-makefile -y am5.yaml -p hpcme.2023 -t prod
+  fre make create-makefile -y null_model.yaml -p hpcme.2023 -t prod
 
   # Create Dockerfile
-  fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod
+  fre make create-dockerfile -y null_model.yaml -p hpcme.2023 -t prod
 
   # Create and run the Dockerfile
-  fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod --execute
+  fre make create-dockerfile -y null_model.yaml -p hpcme.2023 -t prod --execute
 
 4. Run all of fremake:
 
-Currently, run-fremake kicks off the compilation automatically; no ``--execute`` option needed.
+`run-fremake` kicks off the compilation automatically
 
 .. code-block::
 
-  # Bare-metal
-  fre make run-fremake -y am5.yaml -p ncrc5.intel23 -t prod
+  # Bare-metal: create and run checkout script, create makefile, create compile script
+  fre make run-fremake -y null_model.yaml -p ncrc5.intel23 -t prod
 
-  # Container
-  fre make run-fremake -y am5.yaml -p hpcme.2023 -t prod -npc
+  # Bare-metal: create and run checkout script, create makefile, create and run compile script
+  fre make run-fremake -y null_model.yaml -p ncrc5.intel23 -t prod --execute
+
+  # Container: create checkout script, makefile, and dockerfile
+  fre make run-fremake -y null_model.yaml -p hpcme.2023 -t prod -npc
+
+  # Container: create checkout script, makefile, create and run dockerfile to build container
+  fre make run-fremake -y null_model.yaml -p hpcme.2023 -t prod -npc --execute

docs/usage/yaml_framework.rst

@@ -1,4 +1,4 @@
-In order to utilize FRE 2024.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. 
+In order to utilize these FRE tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. 
 
 Yaml Formatting
 ----------
@@ -53,14 +53,15 @@ Where each dash indicates a list.
 
   *ReusableVariable
 
-7. If the reusable variable must be combined with other strings, the **`!join`** constructor is used. Example: 
+7. If the reusable variable must be combined with other strings, the **`!join`** constructor is used. Simplified example: 
 
 .. code-block:: 
 
-  &version "2024.01"
-  &stem !join [FRE/, *version]
+  &name "experiment-name"
+  ...
+  pp_dir: !join [/archive/$USER/, *name, /, pp]
 
-In this example, the reuasble variable ``stem`` will be parsed as ``FRE/2024.01``.
+In this example, the variable ``pp_dir`` will be parsed as ``/archive/$USER/experiment-name/pp``.
 
 Model Yaml
 ----------

docs/what-is-fre.rst

@@ -4,7 +4,7 @@ What is FRE?
 
 FRE, the FMS Runtime Environment, is the companion runtime workflow for FMS-based climate and earth system models, and contains scripts and batch job handlers to compile models, run experiments, and postprocess and analyze the output. Developed around 2004 by GFDL's Modeling System Division, FRE was developed primarily in one repository ("fre-commands", https://github.com/NOAA-GFDL/FRE), used subtools in another repository (FRE-NCtools, https://github.com/NOAA-GFDL/fre-nctools), and was deployed using a set of Environment Modules (https://gitlab.gfdl.noaa.gov/fre/modulefiles). Originally, the major releases of FRE were rivers (Arkansas, Bronx) and the minor releases were numbers. In practice, though, the "Bronx" release name was retained and the number has been incremented over the years. e.g. Bronx-23 is the latest release.
 
-Over the last couple years, MSD's workflow team has reengineered the compiling and postprocessing parts of FRE, in a modern python and Cylc-based ecosystem (running experiments is not yet possible with this new FRE; stay tuned). Following a semantic versioning adopted in other FMS repositories, the reengineered FRE is versioned with a year and incrementing two-digit number. e.g. the first release of 2024 is 2024.01, the second 2024.02, and the first release next year will be 2025.01. (Optional minor releases are also available in the scheme; e.g. 2024.01.01 would be the first minor/patch release after 2024.01.) This version is used as tags in FRE repositories and in the corresponding conda (and in the future, container) release, and can be retrieved from ``fre --version``.
+Over the last couple years, MSD's workflow team has reengineered the compiling and postprocessing parts of FRE, in a modern python and Cylc-based ecosystem (running experiments is not yet possible with this new FRE; stay tuned). Following a semantic versioning adopted in other FMS repositories, the reengineered FRE is versioned with a year and incrementing two-digit number. e.g. the first release of 2025 is 2025.01, the second 2025.02, etc. (Optional minor releases are also available in the scheme; e.g. 2025.01.01 would be the first minor/patch release after 2025.01.) This version is used as tags in FRE repositories and in the corresponding conda (and in the future, container) release, and can be retrieved from ``fre --version``.
 
 fre-cli (this repository) can be considered a successor to the FRE Bronx “fre-commands” repository, which primarily contains user-facing tools and subtools. fre-workflows (https://github.com/NOAA-GFDL/fre-workflows) is a companion repository containing workflow definitions that can be run by the Cylc workflow engine. It contains workflow-specific elements previously in FRE Bronx, and allows flexibility to support multiple and more complex workflows. The two new FRE repositories are versioned with the same approach, and updates will be released together for some time to ensure compatibility.
 

environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - jsonschema
   - noaa-gfdl::fre-nctools
   - noaa-gfdl::catalogbuilder
+  - noaa-gfdl::analysis_scripts
   - conda-forge::nccmp
   - conda-forge::cylc-flow>=8.2.0
   - conda-forge::cylc-rose

fre/analysis/freanalysis.py

@@ -0,0 +1,64 @@
+from analysis_scripts import available_plugins
+import click
+
+from .subtools import install_analysis_package, list_plugins, run_analysis, \
+                      uninstall_analysis_package
+
+
+@click.group(help=click.style(" - access fre analysis subcommands", fg=(250, 154, 90)))
+def analysis_cli():
+    """Entry point to fre analysis click commands."""
+    pass
+
+
+@analysis_cli.command()
+@click.option("--url", type=str, required=True, help="URL of the github repository.")
+@click.option("--name", type=str, required=False, help="Subdirectory to pip install.")
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def install(url, name, library_directory):
+    """Installs an analysis package."""
+    install_analysis_package(url, name, library_directory)
+
+
+@analysis_cli.command()
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def list(library_directory):
+    """List available plugins."""
+    plugins = list_plugins(library_directory)
+    if plugins:
+        print("Installed analysis packages:\n")
+        for plugin in plugins:
+            print(plugin)
+    else:
+        print("No analysis packages found.")
+
+
+@analysis_cli.command()
+@click.option("--name", type=str, required=True, help="Name of the analysis script.")
+@click.option("--catalog", type=str, required=True, help="Path to the data catalog.")
+@click.option("--output-directory", type=str, required=True,
+              help="Path to the output directory.")
+@click.option("--output-yaml", type=str, required=True, help="Path to the output yaml.")
+@click.option("--experiment-yaml", type=str, required=True, help="Path to the experiment yaml.")
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def run(name, catalog, output_directory, output_yaml, experiment_yaml,
+        library_directory):
+    """Runs the analysis script and writes the paths to the created figures to a yaml file."""
+    run_analysis(name, catalog, output_directory, output_yaml, experiment_yaml,
+                 library_directory)
+
+
+@analysis_cli.command()
+@click.option("--name", type=str, required=True, help="Name of package to uninstall.")
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def uninstall(name, library_directory):
+    """Uninstall an analysis package."""
+    uninstall_analysis_package(name, library_directory)
+
+
+if __name__ == "__main__":
+    analysis_cli()