for restart

SFS_baseline-c96/SFS.yaml

@@ -3,18 +3,24 @@ experiment:
   mode: forecast-only
 
 arguments:
-  pslot: {{ 'pslot' | getenv }}
   app: S2S
+  pslot: {{ 'pslot' | getenv }}
+  maxtries: 1
+  cyclethrottle: 1
+  taskthrottle: 100
   resdetatmos: 96
   resensatmos: 96
   resdetocean: 1.0
   nens: 10
   gfs_cyc: 1
-  start: cold
+  start: restart
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 1994050100
-  edate: 1994060100
+ #edate: 1994053100
+ #idate: 1994053100
+  rdate: 1994053100
+  edate: 1994070100
   yaml: {{ HOMEgfs }}/SFS_baseline-c96/SFS_options.yaml
 
 skip_ci_on_hosts:

SFS_baseline-c96/SFS_options.yaml

@@ -7,11 +7,18 @@ base:
   DO_BUFRSND: "NO"
   DO_GEMPAK: "NO"
   DO_AWIPS: "NO"
+  DO_RESTART: "YES"
   KEEPDATA: "YES"
   DO_EXTRACTVARS: "NO"
-  FHMAX_GFS: 744
-  FHMAX_HF_GFS: 744
+  FHMAX: 1440
+  FHMIN: 744
+  FHMAX_GFS: 1464
+  FHMIN_GFS: 744
+  FHMAX_HF_GFS: 1464
   FHOUT_HF_GFS: 24
+  FHMAX_GFS: 1440
+  FHOUT: 24
+  FHOUT_HF: 24
   FHOUT_GFS: 24
   FHOUT_OCN_GFS: 24
   FHOUT_ICE_GFS: 24

parm/config/gefs/config.base

@@ -237,9 +237,13 @@ else
     export OFFSET_START_HOUR=0
 fi
 
+export DO_RESTART=@DO_RESTART@
+if [[ "${DO_RESTART:-NO}" == "YES" ]]; then
+    export OFFSET_START_HOUR="@FHMIN_GFS@"
+fi
 
 # GFS output and frequency
-export FHMIN_GFS=0
+export FHMIN_GFS="@FHMIN_GFS@"
 export FHMAX_GFS="@FHMAX_GFS@"
 # Intermediate times to stop forecast when running in segments
 breakpnts="@FCST_BREAKPOINTS@"
@@ -269,8 +273,8 @@ export FHOUT_OCN=${FHOUT_OCN_GFS}
 export FHOUT_ICE=${FHOUT_ICE_GFS}
 
 # GFS restart interval in hours
-export restart_interval_gfs=600
-export restart_interval_enkfgfs=600
+export restart_interval_gfs=1464
+export restart_interval_enkfgfs=1464
 # NOTE: Do not set this to zero.  Instead set it to $FHMAX_GFS
 # TODO: Remove this variable from config.base and reference from config.fcst
 # TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used

parm/config/gefs/config.fcst

@@ -257,7 +257,7 @@ else
 fi
 
 # Write gfs restart files to rerun fcst from any break point
-export restart_interval=${restart_interval_gfs:-12}
+export restart_interval=${restart_interval_gfs:-1464}
 
 # Turn off dry mass adjustment in GFS
 export adjust_dry_mass=".false."

parm/config/gefs/config.stage_ic

@@ -12,6 +12,8 @@ export BASE_IC="@BASE_IC@" # Platform home for staged ICs
 
 export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2"
 
+export DO_RESTART="YES"
+
 # Set ICSDIR
 
 if [[ -z "${ICSDIR}" ]] ; then

scripts/exglobal_stage_ic.py

@@ -22,7 +22,7 @@ def main():
             'assim_freq', 'current_cycle', 'previous_cycle',
             'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR',
             'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR',
-            'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST']
+            'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'DO_RESTART']
 
     stage_dict = AttrDict()
     for key in keys:

ush/forecast_det.sh

@@ -5,6 +5,9 @@
 UFS_det(){
   echo "SUB ${FUNCNAME[0]}: Run type determination for UFS"
 
+  echo "COMIN_ATMOS_RESTART_PREV: $COMIN_ATMOS_RESTART_PREV"
+  echo "model_start_date_current_cycle: $model_start_date_current_cycle"
+
   # Determine if the current cycle is a warm start (based on the availability of restarts)
   if [[ -f "${COMIN_ATMOS_RESTART_PREV}/${model_start_date_current_cycle:0:8}.${model_start_date_current_cycle:8:2}0000.coupler.res" ]]; then
     warm_start=".true."
@@ -31,7 +34,7 @@ UFS_det(){
   fi
 
   # Lets assume this is was not run before and hence this is not a RERUN
-  RERUN="NO"
+  RERUN="YES"
 
   # RERUN is only available for RUN=gfs|gefs It is not available for RUN=gdas|enkfgdas|enkfgfs
   if [[ "${RUN}" =~ "gdas" ]] || [[ "${RUN}" == "enkfgfs" ]]; then

ush/forecast_postdet.sh

@@ -1,5 +1,8 @@
 #! /usr/bin/env bash
 
+ RERUN=YES
+ RERUN_DATE=1994053100
+
 # Disable variable not used warnings
 # shellcheck disable=SC2034
 FV3_postdet() {
@@ -115,6 +118,7 @@ FV3_postdet() {
 
       local restart_fhr
       restart_fhr=$(nhour "${RERUN_DATE}" "${current_cycle}")
+      IAU_OFFSET=${IAU_OFFSET:-0}
       IAU_FHROT=$((IAU_OFFSET + restart_fhr))
       if [[ "${DOIAU}" == "YES" ]]; then
         IAUFHRS=-1

ush/parsing_namelists_FV3.sh

@@ -1,5 +1,8 @@
 #! /usr/bin/env bash
 
+#Temporary test variables
+print_memory_usage=${print_memory_usage:-".false."}
+
 # parsing namelist of FV3, diag_table, etc.
 
 # Disable variable not used warnings

ush/parsing_ufs_configure.sh

@@ -39,7 +39,7 @@ if [[ "${cpl}" = ".true." ]]; then
   local coupling_interval_slow_sec="${CPL_SLOW}"
 fi
 
-local WRITE_ENDOFRUN_RESTART=.false.
+local WRITE_ENDOFRUN_RESTART=.true.
 
 if [[ "${cplflx}" = ".true." ]]; then
 

ush/python/pygfs/task/stage_ic.py

@@ -53,12 +53,24 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None:
         # Add the os.path.exists function to the dict for yaml parsing
         stage_dict['path_exists'] = os.path.exists
 
+        if 'DO_RESTART' in stage_dict.keys():
+            print("stage_dict['DO_RESTART'] = ", stage_dict['DO_RESTART'])
+            if stage_dict['DO_RESTART']:
+                need_stage_ic = False
+            else:
+                need_stage_ic = True
+        else:
+            need_stage_ic = True
+
+        print('in execute_stage: need_stage_ic: ', need_stage_ic)
+
         # Add the glob.glob function for capturing filenames
         stage_dict['glob'] = glob.glob
 
         # Parse stage yaml to get list of files to copy
         stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False)
 
         # Copy files to ROTDIR
-        for key in stage_set.keys():
-            FileHandler(stage_set[key]).sync()
+        if need_stage_ic:
+            for key in stage_set.keys():
+                FileHandler(stage_set[key]).sync()

workflow/create_experiment.py

@@ -94,15 +94,32 @@ def input_args():
         setup_expt_args.append("--overwrite")
 
     logger.info(f"Call: setup_expt.main()")
+
     logger.debug(f"setup_expt.py {' '.join(setup_expt_args)}")
+
     setup_expt.main(setup_expt_args)
 
     # Create a list of arguments to setup_xml.py
     experiment_dir = Path.absolute(Path.joinpath(
         Path(testconf.arguments.expdir), Path(testconf.arguments.pslot)))
 
-    setup_xml_args = [str(experiment_dir)]
+    setup_xml_args = ['--expdir', str(experiment_dir)]
+
+    for n in range(len(setup_expt_args)):
+        key = setup_expt_args[n]
+        if (key == '--maxtries'):
+            setup_xml_args.append(key)
+            setup_xml_args.append(setup_expt_args[n+1])
+        elif (key == '--cyclethrottle'):
+            setup_xml_args.append(key)
+            setup_xml_args.append(setup_expt_args[n+1])
+        elif (key =='--taskthrottle'):
+            setup_xml_args.append(key)
+            setup_xml_args.append(setup_expt_args[n+1])
+
+   #print('setup_xml_args:', setup_xml_args)
 
     logger.info(f"Call: setup_xml.main()")
     logger.debug(f"setup_xml.py {' '.join(setup_xml_args)}")
+   #print('setup_xml_args:', setup_xml_args)
     setup_xml.main(setup_xml_args)

workflow/rocoto/tasks.py

@@ -148,7 +148,7 @@ def _get_forecast_hours(run, config, component='atmos') -> List[str]:
             local_config['FHOUT_GFS'] = config['FHOUT_ICE_GFS']
             local_config['FHOUT'] = config['FHOUT_ICE']
 
-        fhmin = local_config['FHMIN']
+        fhmin = local_config['FHMIN_GFS']
 
         # Get a list of all forecast hours
         fhrs = []
@@ -161,7 +161,18 @@ def _get_forecast_hours(run, config, component='atmos') -> List[str]:
             fhout = local_config['FHOUT_GFS']
             fhmax_hf = local_config['FHMAX_HF_GFS']
             fhout_hf = local_config['FHOUT_HF_GFS']
+
+            print('fhmin = ', fhmin)
+            print('fhmax = ', fhmax)
+            print('fhout = ', fhout)
+            print('fhmax_hf = ', fhmax_hf)
+            print('fhout_hf = ', fhout_hf)
+
+            if (fhmax_hf < 1):
+                fhmax_hf = fhmax
+
             fhrs_hf = range(fhmin, fhmax_hf + fhout_hf, fhout_hf)
+            print('fhrs_hf = ', fhrs_hf)
             fhrs = list(fhrs_hf) + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout))
 
         return fhrs

workflow/setup_expt.py

@@ -100,21 +100,35 @@ def edit_baseconfig(host, inputs, yaml_dict):
     extend_dict = get_template_dict(host.info)
     tmpl_dict = dict(tmpl_dict, **extend_dict)
 
+    is_warm_start = ".false."
+    is_restart = ".false."
     if inputs.start in ["warm"]:
         is_warm_start = ".true."
     elif inputs.start in ["cold"]:
         is_warm_start = ".false."
+    elif inputs.start in ["restart"]:
+        is_restart = ".true."
+
+   #print('inputs: ', inputs)
+
+    if hasattr(inputs, 'rdate'):
+        inputs.idate = inputs.rdate
 
     extend_dict = dict()
     extend_dict = {
         "@PSLOT@": inputs.pslot,
+        "@MAXTRIES@": f"{int(inputs.maxtries):03d}",
+        "@CYCLETHROTTLE@": f"{int(inputs.cyclethrottle):03d}",
+        "@TASKTHROTTLE@": f"{int(inputs.taskthrottle):03d}",
         "@SDATE@": datetime_to_YMDH(inputs.idate),
+        "@RDATE@": datetime_to_YMDH(inputs.rdate),
         "@EDATE@": datetime_to_YMDH(inputs.edate),
         "@CASECTL@": f'C{inputs.resdetatmos}',
         "@OCNRES@": f"{int(100.*inputs.resdetocean):03d}",
         "@EXPDIR@": inputs.expdir,
         "@COMROOT@": inputs.comroot,
         "@EXP_WARM_START@": is_warm_start,
+        "@EXP_RESTART@": is_restart,
         "@MODE@": inputs.mode,
         "@gfs_cyc@": inputs.gfs_cyc,
         "@APP@": inputs.app,
@@ -199,15 +213,22 @@ def _common_args(parser):
                             type=str, required=False, default=os.getenv('HOME'))
         parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!',
                             required=True, type=lambda dd: to_datetime(dd))
+        parser.add_argument('--rdate', help='restart date experiment', required=False, type=lambda dd: to_datetime(dd))
         parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd))
         parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='')
         parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)',
                             action='store_true', required=False)
+        parser.add_argument('--maxtries', help='maximum number of retries', type=int,
+                            default=2, required=False)
+        parser.add_argument('--cyclethrottle', help='maximum number of concurrent cycles', type=int,
+                            default=3, required=False)
+        parser.add_argument('--taskthrottle', help='maximum number of concurrent tasks', type=int,
+                            default=25, required=False)
         return parser
 
     def _gfs_args(parser):
-        parser.add_argument('--start', help='restart mode: warm or cold', type=str,
-                            choices=['warm', 'cold'], required=False, default='cold')
+        parser.add_argument('--start', help='restart mode: warm, cold, or restart', type=str,
+                            choices=['warm', 'cold', 'restart'], required=False, default='cold')
         parser.add_argument('--run', help='RUN to start the experiment',
                             type=str, required=False, default='gdas')
         # --configdir is hidden from help
@@ -238,8 +259,8 @@ def _gfs_or_gefs_forecast_args(parser):
         return parser
 
     def _gefs_args(parser):
-        parser.add_argument('--start', help='restart mode: warm or cold', type=str,
-                            choices=['warm', 'cold'], required=False, default='cold')
+        parser.add_argument('--start', help='restart mode: warm, cold, or restart', type=str,
+                            choices=['warm', 'cold', 'restart'], required=False, default='cold')
         parser.add_argument('--configdir', help=SUPPRESS, type=str, required=False,
                             default=os.path.join(_top, 'parm/config/gefs'))
         parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False,
@@ -347,6 +368,8 @@ def main(*argv):
     user_inputs = input_args(*argv)
     host = Host()
 
+   #print('user_inputs = ', user_inputs)
+
     validate_user_request(host, user_inputs)
 
     # Determine ocean resolution if not provided

workflow/setup_xml.py

@@ -25,9 +25,6 @@ def input_args(*argv):
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
     # Common arguments across all modes
-    parser.add_argument('expdir', help='full path to experiment directory containing config files',
-                        type=str, default=os.environ['PWD'])
-
     parser.add_argument('--maxtries', help='maximum number of retries', type=int,
                         default=2, required=False)
     parser.add_argument('--cyclethrottle', help='maximum number of concurrent cycles', type=int,
@@ -36,8 +33,10 @@ def input_args(*argv):
                         default=25, required=False)
     parser.add_argument('--verbosity', help='verbosity level of Rocoto', type=int,
                         default=10, required=False)
+    parser.add_argument('--expdir', help='full path to experiment directory containing config files',
+                        type=str, default=os.environ['PWD'])
 
-    return parser.parse_args(argv[0][0] if len(argv[0]) else None)
+    return parser.parse_args(list(*argv[0]) if len(argv[0]) else None)
 
 
 def check_expdir(cmd_expdir, cfg_expdir):
@@ -51,12 +50,19 @@ def check_expdir(cmd_expdir, cfg_expdir):
 
 def main(*argv):
 
+   #print('in setup_xml.py argv = ', argv)
+
     user_inputs = input_args(argv)
+
+   #print('in setup_xml.py user_inputs = ', user_inputs)
+
     rocoto_param_dict = {'maxtries': user_inputs.maxtries,
                          'cyclethrottle': user_inputs.cyclethrottle,
                          'taskthrottle': user_inputs.taskthrottle,
                          'verbosity': user_inputs.verbosity}
 
+   #print('in setup_xml.py rocoto_param_dict = ', rocoto_param_dict)
+
     cfg = Configuration(user_inputs.expdir)
 
     base = cfg.parse_config('config.base')