Use offset aware times and adjust based on testing on PPI

examples/surfex_namelists.yml

@@ -7841,14 +7841,14 @@ soda_isba_ekf:
     NVAR: '${SURFEX#SODA#NVAR}'
     XALPHA: '${SURFEX#ASSIM#ISBA#EKF#XALPHA}'
     XSCALE_Q: '${SURFEX#ASSIM#ISBA#EKF#XSCALE_Q}'
-    NNVC: '${SURFEX#ASSIM#ISBA#EKF#NNCV}'
+    NNCV: '${SURFEX#ASSIM#ISBA#EKF#NNCV}'
     CVAR_M: '${SURFEX#ASSIM#ISBA#EKF#CVAR_M}'
 
 soda_isba_enkf:
   NAM_VAR:
     NIVAR: '${SURFEX#SODA#NIVAR}'
     NVAR: '${SURFEX#SODA#NVAR}'
-    NNVC: '${SURFEX#ASSIM#ISBA#ENKF#NNCV}'
+    NNCV: '${SURFEX#ASSIM#ISBA#ENKF#NNCV}'
     CVAR_M: '${SURFEX#ASSIM#ISBA#ENKF#CVAR_M}'
     NENS_M: '${SURFEX#ASSIM#ISBA#ENKF#NENS_M}'
 

pysurfex/datetime_utils.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """Implement helper routines to deal with dates and times."""
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, timezone
 
 
 # TODO use ISO times
@@ -14,8 +14,7 @@ def as_datetime(dtg):
         fmt = "%Y%m%d%H%M%S"
     else:
         raise RuntimeError(f"dtg={dtg} len(dtg) is {len(dtg)}")
-
-    return datetime.strptime(dtg, fmt)
+    return datetime.strptime(dtg, fmt).replace(tzinfo=timezone.utc)
 
 
 def as_datetime_string(dtg):
@@ -36,7 +35,7 @@ def fromtimestamp(validtime):
 
 def utcfromtimestamp(epochtime):
     """Convert timestamp to validtime."""
-    return datetime.utcfromtimestamp(epochtime)
+    return datetime.utcfromtimestamp(epochtime).replace(tzinfo=timezone.utc)
 
 
 def isdatetime(obj):
@@ -48,4 +47,4 @@ def as_datetime_args(year=None, month=None, day=None, hour=0, minute=0, second=0
     """Set datetime object from args."""
     return datetime(
         year=year, month=month, day=day, hour=hour, minute=minute, second=second
-    )
+    ).replace(tzinfo=timezone.utc)

pysurfex/file.py

@@ -915,7 +915,13 @@ def field(self, var, validtime=None):
 
             if time_in_file is not None:
                 if validtime != time_in_file:
-                    logging.error("%s %s", time_in_file, validtime)
+                    logging.error(
+                        "time_in_file=%s validtime=%s %s %s",
+                        time_in_file,
+                        validtime,
+                        type(time_in_file),
+                        type(validtime),
+                    )
                     raise RuntimeError("Mismatch in times in file and the wanted time")
 
         geo_in = self.get_geo()

pysurfex/namelist.py

@@ -24,6 +24,7 @@ def __init__(self, program, config, definitions, assemble=None, consistency=True
         self.program = program
         self.config = config
         self.nldict = definitions
+        """
         macros_defs = {
             "CPGDFILE": "SURFEX#IO#CPGDFILE",
             "CPREPFILE": "SURFEX#IO#CPREPFILE",
@@ -43,7 +44,10 @@ def __init__(self, program, config, definitions, assemble=None, consistency=True
             vmacro = config.get_setting(setting)
             logging.debug("Mapping macro %s = %s", macro, vmacro)
             if vmacro is not None:
+                if isinstance(vmacro, tuple):
+                    vmacro = list(vmacro)
                 macros.update({macro: vmacro})
+        """
 
         nobstype = 0
         if program == "soda" or program == "offline":
@@ -113,6 +117,8 @@ def _flatten_dict_gen(dic, parent_key, sep):
                 if isinstance(val, collections.abc.MutableMapping):
                     yield from flatten_dict(val, new_key, sep=sep).items()
                 else:
+                    if isinstance(val, tuple):
+                        val = list(val)
                     yield new_key, val
 
         def flatten_dict(

tests/unit/test_binary_input_data.py

@@ -123,7 +123,7 @@ def f90ml_namelist(tmp_path_factory):
             LECOSG = .True.
         /
         &NAM_IO_OFFLINE
-            CSURFFILETYPE = "FA"
+            CSURF_FILETYPE = "FA"
         /
         &NAM_DATA_ISBA
             NTIME = 36
@@ -207,19 +207,19 @@ def test_new_binary_input(f90ml_namelist, input_binary_data_file):
     logging.debug("binary_data=%s", binary_data.data)
     assert (
         binary_data.data["filename_albnir_soil_2_0115.dir"]
-        == "/ecoclimap/ALB_SAT/ALB_SAT_NI_0115_c.dir"
+        == "/ecoclimap/ALB/ALB_SAT/ALB_SAT_NI_0115_c.dir"
     )
     assert (
         binary_data.data["filename_albnir_soil_20_1225.dir"]
-        == "/ecoclimap/ALB_SAT/ALB_SAT_NI_1225_c.dir"
+        == "/ecoclimap/ALB/ALB_SAT/ALB_SAT_NI_1225_c.dir"
     )
     assert (
         binary_data.data["filename_albvis_soil_2_0115.dir"]
-        == "/ecoclimap/ALB_SAT/ALB_SAT_VI_0115_c.dir"
+        == "/ecoclimap/ALB/ALB_SAT/ALB_SAT_VI_0115_c.dir"
     )
     assert binary_data.data["filename_h_tree_1.dir"] == "/ecoclimap/HT/new_ht_c.dir"
     assert binary_data.data["filename_h_tree_20.dir"] == "/ecoclimap/HT/new_ht_c.dir"
-    assert binary_data.data["gmted2010file.dir"] == "/gmted/gmted2010.dir"
+    assert binary_data.data["gmted2010file.dir"] == "/climdir/gmted2010.dir"
 
     # Prep
     input_data_copy = input_data.copy()

tests/unit/test_converter.py

@@ -1,9 +1,8 @@
 """Test converter."""
-from datetime import datetime
-
 import numpy as np
 
 from pysurfex.cache import Cache
+from pysurfex.datetime_utils import as_datetime
 from pysurfex.geo import get_geo_object
 from pysurfex.read import ConvertedInput, Converter
 
@@ -32,7 +31,7 @@ def test_converter_meps_nc(conf_proj_2x3_dict, data_thredds_nc_file):
     defs = config[fileformat]
     converter_conf = config[var][fileformat]["converter"]
 
-    validtime = datetime(year=2020, month=2, day=20, hour=6)
+    validtime = as_datetime("2020022006")
     cache = Cache(7200)
     converter = Converter(converter, validtime, defs, converter_conf, fileformat)
     field = ConvertedInput(my_geo, var, converter).read_time_step(validtime, cache)