Daycli update (#13)

* Update to column names for daycli and addition of missing columns. * quoting none added to daycli * Addition of valid ranges * Fix of cut and paste error in daycli.json * consistency fixes. * Update to some valid_max to include range up to missing value. Addition of sample file for DAYCLI.
wmo-im · Sep 30, 2024 · ee54925 · ee54925
1 parent 984abe2
commit ee54925
Show file tree

Hide file tree

Showing 2 changed files with 86 additions and 49 deletions.
diff --git a/samples/daycli.csv b/samples/daycli.csv
@@ -0,0 +1,31 @@
+wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,latitude,longitude,station_height_above_msl,temperature_siting_classification,precipitation_siting_classification,averaging_method,year,month,day,precipitation_day_offset,precipitation_hour,precipitation_minute,precipitation_second,precipitation,precipitation_flag,fresh_snow_day_offset,fresh_snow_hour,fresh_snow_minute,fresh_snow_second,fresh_snow_depth,fresh_snow_depth_flag,total_snow_day_offset,total_snow_hour,total_snow_minute,total_snow_second,total_snow_depth,total_snow_depth_flag,thermometer_height,maximum_temperature_day_offset,maximum_temperature_hour,maximum_temperature_minute,maximum_temperature_second,maximum_temperature,maximum_temperature_flag,minimum_temperature_day_offset,minimum_temperature_hour,minimum_temperature_minute,minimum_temperature_second,minimum_temperature,minimum_temperature_flag,average_temperature_day_offset,average_temperature_hour,average_temperature_minute,average_temperature_second,average_temperature,average_temperature_flag
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,1,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,278.15,0,0,7,0,1,273.75,0,0,0,0,1,276.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,2,0,7,0,1,1.5,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,279.25,0,0,7,0,1,273.75,0,0,0,0,1,275.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,3,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.75,0,0,7,0,1,274.25,0,0,0,0,1,280.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,4,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.75,0,0,7,0,1,274.85,0,0,0,0,1,281.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,5,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,279.85,0,0,0,0,1,287.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,6,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,299.85,0,0,7,0,1,277.55,0,0,0,0,1,286.95,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,7,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,298.15,0,0,7,0,1,280.95,0,0,0,0,1,287.5,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,8,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,271.55,0,0,0,0,1,281.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,9,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.05,0,0,7,0,1,272.15,0,0,0,0,1,278.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,10,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,273.75,0,0,0,0,1,281.55,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,11,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,286.45,0,0,7,0,1,268.85,0,0,0,0,1,277.85,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,12,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,266.55,0,0,0,0,1,276.55,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,13,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,274.25,0,0,0,0,1,284.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,14,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,268.85,0,0,0,0,1,281.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,15,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,280.95,0,0,0,0,1,290.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,16,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.85,0,0,0,0,1,288.55,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,17,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,275.95,0,0,7,0,1,266.05,0,0,0,0,1,276.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,18,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,262.15,0,0,0,0,1,271.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,19,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.05,0,0,7,0,1,275.35,0,0,0,0,1,280.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,20,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,274.85,0,0,0,0,1,281.45,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,21,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,269.35,0,0,0,0,1,277.65,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,22,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,272.65,0,0,0,0,1,281.95,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,23,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.25,0,0,0,0,1,283.95,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,24,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,264.95,0,0,0,0,1,281.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,25,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.15,0,0,7,0,1,264.35,0,0,0,0,1,273.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,26,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,278.15,0,0,0,0,1,283.15,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,27,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,271.05,0,0,0,0,1,281.85,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,28,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,271.05,0,0,0,0,1,278.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,29,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,295.35,0,0,7,0,1,277.05,0,0,0,0,1,286.15,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,30,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,289.25,0,0,7,0,1,274.85,0,0,0,0,1,281.65,0
diff --git a/templates/daycli-template.json b/templates/daycli-template.json
@@ -3,26 +3,26 @@
     "metadata": {
         "label": "daycli-template",
         "description": "Mapping template for converting daily climate CSV data to BUFR sequence 307075",
-        "version": "2",
+        "version": "3",
         "author": "David I. Berry",
-        "editor": "",
+        "editor": "David I. Berry",
         "dateCreated": "2023-09-01",
-        "dateModified": "2024-01-12",
-        "id": "4a077105-d52c-458b-867a-1dfb743156d0"
+        "dateModified": "2024-09-27",
+        "id": "2fe51aa5-d98a-4712-9954-45c7075462da"
     },
     "inputShortDelayedDescriptorReplicationFactor": [],
     "inputDelayedDescriptorReplicationFactor": [],
     "inputExtendedDelayedDescriptorReplicationFactor": [],
     "number_header_rows": 1,
     "column_names_row": 1,
-    "wigos_station_identifier": "data:wigos_station_identifier",
+    "quoting": "QUOTE_NONE",
     "header": [
         {"eccodes_key": "edition", "value": "const:4"},
         {"eccodes_key": "masterTableNumber", "value": "const:0"},
         {"eccodes_key": "updateSequenceNumber", "value": "const:0"},
         {"eccodes_key": "dataCategory", "value": "const:0"},
         {"eccodes_key": "internationalDataSubCategory", "value": "const:6"},
-        {"eccodes_key": "masterTablesVersionNumber", "value": "const:38"},
+        {"eccodes_key": "masterTablesVersionNumber", "value": "const:39"},
         {"eccodes_key": "typicalYear", "value": "data:year"},
         {"eccodes_key": "typicalMonth", "value": "data:month"},
         {"eccodes_key": "typicalDay", "value": "data:day"},
@@ -34,60 +34,66 @@
         {"eccodes_key": "unexpandedDescriptors", "value": "array:307075"}
     ],
     "data": [
-        {"eccodes_key": "#1#wigosIdentifierSeries", "value": "data:wsi_series"},
-        {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value": "data:wsi_issuer"},
-        {"eccodes_key": "#1#wigosIssueNumber", "value": "data:wsi_issue"},
-        {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value": "data:wsi_local"},
-        {"eccodes_key": "#1#latitude", "value": "data:lat", "valid_min": "const:-90.0", "valid_max": "const:90.0"},
-        {"eccodes_key": "#1#longitude", "value": "data:lon", "valid_min": "const:-180.0", "valid_max": "const:180.0"},
-        {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value": "data:elev"},
-        {"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "const:0"},
+        {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:wsi_series", "valid_min": "const:0", "valid_max": "const:0"},
+        {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:wsi_issuer", "valid_min": "const:0", "valid_max": "const:65534"},
+        {"eccodes_key": "#1#wigosIssueNumber", "value":"data:wsi_issue_number", "valid_min": "const:0", "valid_max": "const:65534"},
+        {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:wsi_local"},
+        {"eccodes_key": "#1#blockNumber", "value": "data:wmo_block_number", "valid_min": "const:0", "valid_max": "const:99"},
+        {"eccodes_key": "#1#stationNumber", "value": "data:wmo_station_number", "valid_min": "const:0", "valid_max": "const:999"},
+        {"eccodes_key": "#1#latitude", "value": "data:latitude", "valid_min": "const:-90.0", "valid_max": "const:90.0"},
+        {"eccodes_key": "#1#longitude", "value": "data:longitude", "valid_min": "const:-180.0", "valid_max": "const:180.0"},
+        {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:station_height_above_msl", "valid_min": "const:-400", "valid_max": "const:9000"},
+        {"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "data:averaging_method", "valid_min": "const:0", "valid_max":  "const:255"},
+        {"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForTemperature", "value": "data:temperature_siting_classification", "valid_min": "const:0", "valid_max": "const:255"},
+        {"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForPrecipitation", "value": "data:precipitation_siting_classification", "valid_min": "const:0", "valid_max": "const:255"},
         {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1800", "valid_max": "const:2100"},
         {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"},
         {"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"},
-        {"eccodes_key": "#1#timePeriod", "value": "const:0"},
-        {"eccodes_key": "#1#hour", "value": "const:0"},
-        {"eccodes_key": "#1#minute", "value": "const:0"},
-        {"eccodes_key": "#1#minute", "value": "const:0"},
-        {"eccodes_key": "#1#second", "value": "const:0"},
-        {"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:total_precipitation_or_total_water_equivalent"},
-        {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "const:7"},
+        {"eccodes_key": "#1#timePeriod", "value": "data:precipitation_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
+        {"eccodes_key": "#1#hour", "value": "data:precipitation_hour", "valid_min": "const:0", "valid_max": "const:23"},
+        {"eccodes_key": "#1#minute", "value": "data:precipitation_minute", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#1#second", "value": "data:precipitation_second", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:precipitation", "valid_min":  "const:0", "valid_max": "const:2000"},
+        {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "data:precipitation_flag", "valid_min": "const:0", "valid_max": "const:7"},
         {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField->associatedFieldSignificance", "value": "const:5"},
-        {"eccodes_key": "#2#timePeriod", "value": "const:0"},
-        {"eccodes_key": "#2#hour", "value": "const:0"},
-        {"eccodes_key": "#2#minute", "value": "const:0"},
-        {"eccodes_key": "#2#second", "value": "const:0"},
-        {"eccodes_key": "#1#depthOfFreshSnow->associatedField", "value": "const:5"},
+        {"eccodes_key": "#2#timePeriod", "value": "data:fresh_snow_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
+        {"eccodes_key": "#2#hour", "value": "data:fresh_snow_hour", "valid_min": "const:0", "valid_max": "const:23"},
+        {"eccodes_key": "#2#minute", "value": "data:fresh_snow_minute", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#2#second", "value": "data:fresh_snow_second", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#1#depthOfFreshSnow", "value": "data:fresh_snow_depth", "valid_min": "const:0"},
+        {"eccodes_key": "#1#depthOfFreshSnow->associatedField", "value": "data:fresh_snow_depth_flag", "valid_min": "const:0", "valid_max": "const:7"},
         {"eccodes_key": "#1#depthOfFreshSnow->associatedField->associatedFieldSignificance", "value": "const:5"},
-        {"eccodes_key": "#3#timePeriod", "value": "const:0"},
-        {"eccodes_key": "#3#hour", "value": "const:0"},
-        {"eccodes_key": "#3#minute", "value": "const:0"},
-        {"eccodes_key": "#3#second", "value": "const:0"},
-        {"eccodes_key": "#1#totalSnowDepth->associatedField", "value": "const:5"},
+        {"eccodes_key": "#3#timePeriod", "value": "data:total_snow_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
+        {"eccodes_key": "#3#hour", "value": "data:total_snow_hour", "valid_min": "const:0", "valid_max": "const:23"},
+        {"eccodes_key": "#3#minute", "value": "data:total_snow_minute", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#3#second", "value": "data:total_snow_second", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#1#totalSnowDepth", "value": "data:total_snow_depth", "valid_min": "const:0"},
+        {"eccodes_key": "#1#totalSnowDepth->associatedField", "value": "data:total_snow_depth_flag", "valid_min": "const:0", "valid_max": "const:7"},
         {"eccodes_key": "#1#totalSnowDepth->associatedField->associatedFieldSignificance", "value": "const:5"},
-        {"eccodes_key": "#4#timePeriod", "value": "const:0"},
-        {"eccodes_key": "#4#hour", "value": "const:0"},
-        {"eccodes_key": "#4#minute", "value": "const:0"},
-        {"eccodes_key": "#4#second", "value": "const:0"},
+        {"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:thermometer_height", "valid_min": "const:0", "valid_max": "const:655.35"},
+        {"eccodes_key": "#4#timePeriod", "value": "data:maximum_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
+        {"eccodes_key": "#4#hour", "value": "data:maximum_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"},
+        {"eccodes_key": "#4#minute", "value": "data:maximum_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#4#second", "value": "data:maximum_temperature_second", "valid_min": "const:0", "valid_max": "const:59"},
         {"eccodes_key": "#1#firstOrderStatistics", "value": "const:2"},
-        {"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature_at_height_and_over_period_specified", "scale": "const:0", "offset": "const:273.15"},
-        {"eccodes_key": "#1#airTemperature->associatedField", "value": "const:7"},
+        {"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"},
+        {"eccodes_key": "#1#airTemperature->associatedField", "value": "data:maximum_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"},
         {"eccodes_key": "#1#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"},
-        {"eccodes_key": "#5#timePeriod", "value": "const:0"},
-        {"eccodes_key": "#5#hour", "value": "const:0"},
-        {"eccodes_key": "#5#minute", "value": "const:0"},
-        {"eccodes_key": "#5#second", "value": "const:0"},
+        {"eccodes_key": "#5#timePeriod", "value": "data:minimum_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
+        {"eccodes_key": "#5#hour", "value": "data:minimum_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"},
+        {"eccodes_key": "#5#minute", "value": "data:minimum_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#5#second", "value": "data:minimum_temperature_second", "valid_min": "const:0", "valid_max": "const:59"},
         {"eccodes_key": "#2#firstOrderStatistics", "value": "const:3"},
-        {"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature_at_height_and_over_period_specified", "scale": "const:0", "offset": "const:273.15"},
-        {"eccodes_key": "#2#airTemperature->associatedField", "value": "const:7"},
+        {"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"},
+        {"eccodes_key": "#2#airTemperature->associatedField", "value": "data:minimum_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"},
         {"eccodes_key": "#2#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"},
-        {"eccodes_key": "#6#timePeriod", "value": "const:0"},
-        {"eccodes_key": "#6#hour", "value": "const:0"},
-        {"eccodes_key": "#6#minute", "value": "const:0"},
-        {"eccodes_key": "#6#second", "value": "const:0"},
+        {"eccodes_key": "#6#timePeriod", "value": "data:average_temperature_day_offset", "valid_min": "const:-1", "valid_max": "const:0"},
+        {"eccodes_key": "#6#hour", "value": "data:average_temperature_hour", "valid_min": "const:0", "valid_max": "const:23"},
+        {"eccodes_key": "#6#minute", "value": "data:average_temperature_minute", "valid_min": "const:0", "valid_max": "const:59"},
+        {"eccodes_key": "#6#second", "value": "data:average_temperature_second", "valid_min": "const:0", "valid_max": "const:59"},
         {"eccodes_key": "#3#firstOrderStatistics", "value": "const:4"},
-        {"eccodes_key": "#3#airTemperature", "value": "data:mean_air_temperature", "scale": "const:0", "offset": "const:273.15"},
-        {"eccodes_key": "#3#airTemperature->associatedField", "value": "const:7"},
+        {"eccodes_key": "#3#airTemperature", "value": "data:average_temperature", "valid_min": "const:183.15", "valid_max": "const:343.15"},
+        {"eccodes_key": "#3#airTemperature->associatedField", "value": "data:average_temperature_flag", "valid_min": "const:0", "valid_max": "const:7"},
         {"eccodes_key": "#3#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}
     ]
 }