Update timestamp to identify all duplicates with information flag, rather than just non-equal ones

io_utils.py

@@ -196,6 +196,7 @@ def flag_write(outfilename: str, df: pd.DataFrame, diagnostics: bool = False) ->
             this_var_data = df[var].fillna(MDI).to_numpy().astype(float)
             this_var_data = np.ma.masked_where(this_var_data == MDI, this_var_data)
 
+            # write out for all tests, regardless if set for this variable or not
             for test in QC_TESTS.keys():
                 locs = flags[flags.str.contains(test)]
 
@@ -219,7 +220,7 @@ def flag_write(outfilename: str, df: pd.DataFrame, diagnostics: bool = False) ->
             logging.info(f"{var} - {flagged.shape[0]}")
             if diagnostics:
                 print(f"{var} - {flagged.shape[0]}")
-                print(f"{var} - {flagged.shape[0]/np.ma.count(this_var_data)}")
+                print(f"{var} - {100*flagged.shape[0]/np.ma.count(this_var_data):.1f}%")
 
 
     return # flag_write

plot_map_of_flagging_rates.py

@@ -14,16 +14,16 @@
 
 
 TESTS_FOR_VARS = {"temperature": ["All", "C", "D", "E", "F", "H", "K", "L", "N", "T",
-                                  "S", "U", "V", "W", "d", "o", "x", "y",],
+                                  "S", "U", "V", "W", "d", "o", "x", "y", "2",],
                   "dew_point_temperature": ['All', 'C', 'D', 'E', 'F', 'H', 'K', 'L', 'N', 'S',
-                                            'T', 'V', 'W', 'd', 'h', 'n', 'o', 'x', 'y'],
+                                            'T', 'V', 'W', 'd', 'h', 'n', 'o', 'x', 'y', "2",],
                   "sea_level_pressure" : ['All', 'D', 'E', 'F', 'H', 'K', 'L', 'N', 'S',
-                                          'T', 'V', 'W', 'd', 'o', 'p', 'x', 'y'],
+                                          'T', 'V', 'W', 'd', 'o', 'p', 'x', 'y', "2",],
                   "station_level_pressure" : ['All', 'D', 'E', 'F', 'H', 'K', 'L', 'N',
-                                              'S', 'T', 'V', 'd', 'o', 'p', 'x', 'y'],
+                                              'S', 'T', 'V', 'd', 'o', 'p', 'x', 'y', "2",],
                   "wind_speed" : ['All', 'E', 'H', 'K', 'L', 'N', 'S', 'T', 'V',
-                                  'W', 'o', 'w', 'x', 'y'],
-                  "wind_direction" : ['All', 'E', 'H', 'K', 'L', 'w', 'x', 'y', "1"]}
+                                  'W', 'o', 'w', 'x', 'y', "2",],
+                  "wind_direction" : ['All', 'E', 'H', 'K', 'L', 'w', 'x', 'y', "1", "2",]}
 
 UNITS = {"" : "%", "_counts" : "cts"}
 

qc_tests/timestamp.py

@@ -69,11 +69,19 @@ def identify_multiple_values(obs_var: utils.Meteorological_Variable, times: np.n
         print(f" Number of identical timestamps in {obs_var.name}: {multiple_obs_at_time.shape[0]}")
 
     if len(multiple_obs_at_time) != 0:
+        # to the observations differ for the entries
         suspect_locs, = np.ma.where(value_diffs[multiple_obs_at_time] != 0)
 
-        # set the first of the obs, then the second which make the diff
-        compressed_flags[multiple_obs_at_time[suspect_locs]] = "T"
-        compressed_flags[multiple_obs_at_time[suspect_locs]+1] = "T"
+        if len(suspect_locs) > 0:
+            # Observations have different values, so not clear which is correct.
+            #   Flag both
+            # set the first of the obs, then the second which make the diff
+            compressed_flags[multiple_obs_at_time[suspect_locs]] = "T"
+            compressed_flags[multiple_obs_at_time[suspect_locs]+1] = "T"
+        else:
+            # Observations have the _same_ value, so add information flag onl
+            compressed_flags[multiple_obs_at_time] = "2"
+            compressed_flags[multiple_obs_at_time+1] = "2"
 
         # Uncompress the flags & insert
         flags = np.array(["" for i in range(obs_var.data.shape[0])])

qc_utils.py

@@ -49,6 +49,7 @@
             "x" : "Excess streak proportion",
             "y" : "Repeated Day streaks",
             "1" : "Wind logical - calm, masked direction",
+            "2" : "Timestamp - identical observation qvalues",
             }
 
 

tests/example_data/Example_flag_file.flg

@@ -40,5 +40,9 @@ temperature : x : 0.0
 temperature : x_counts : 0
 temperature : y : 0.0
 temperature : y_counts : 0
+temperature : 1 : 0.0
+temperature : 1_counts : 0
+temperature : 2 : 0.0
+temperature : 2_counts : 0
 temperature : All : 0.8
 temperature : All_counts : 4

tests/test_qc_tests/test_timestamp.py

@@ -88,10 +88,11 @@ def test_identify_multiple_values_same():
     station.times[3] = station.times[2]
 
     # Even though two time stamps are the same, the values are identical
-    #   (np.ones) so no flag set
+    #   (np.ones) but an information flag still set
     expected = np.array(["" for _ in range(10)])
+    expected[2:4] = "2"
 
-    timestamp.identify_multiple_values(obs_var, station. times, {})
+    timestamp.identify_multiple_values(obs_var, station.times, {})
 
     np.testing.assert_array_equal(obs_var.flags, expected)