Merge branch 'master' into cadeban/add-source-labels

parsers/NED.py

@@ -84,16 +84,18 @@ def _kwh_to_mw(kwh):
     return round((kwh / 1000) * 4, 3)
 
 
-# There is a limit of items we can get per page, so we fetch all possible per page.
+# It seems the API can take max itemPerPage 200. We fetch 192 items per page as this is: (12 types * 4 quaters * 4 hours) = 192
+# If the itemsPerPage is not a multiple of the types the API sometime skips a type, sometimes duplicates a type!
 # The API does not include the last page number in the response, so we need to keep querying until we get an empty response
 def call_api(target_datetime: datetime, forecast: bool = False):
     is_last_page = False
     pageNum = 1
     results = []
     while not is_last_page:
+        # API fetches full day of data, so we add 1 day to validfrom[before] to get todays data
         params = {
             "page": pageNum,
-            "itemsPerPage": 3500,
+            "itemsPerPage": 192,
             "point": NedPoint.NETHERLANDS.value,
             "type[]": [
                 NedType.WIND.value,
@@ -135,7 +137,7 @@ def call_api(target_datetime: datetime, forecast: bool = False):
         results += response.json()
         pageNum += 1
 
-        if response.json() == [] or pageNum > 20:
+        if response.json() == [] or pageNum > 26:
             is_last_page = True
 
     return results
@@ -166,6 +168,13 @@ def format_data(
 
     formatted_production_data = ProductionBreakdownList(logger)
     for _group_key, group_df in df:
+        # Add lag to avoid using data that is not yet complete and remove "future" data
+        if (
+            datetime.fromisoformat(group_df["validfrom"].iloc[0])
+            > (datetime.now(timezone.utc) - timedelta(hours=0.5))
+            and not forecast
+        ):
+            continue
         data_dict = group_df.to_dict(orient="records")
         mix = ProductionMix()
         for data in data_dict:
@@ -185,6 +194,7 @@ def format_data(
             if forecast
             else EventSourceType.measured,
         )
+
     return formatted_production_data
 
 
@@ -199,7 +209,6 @@ def fetch_production(
     target_datetime = target_datetime or datetime.now(timezone.utc)
 
     json_data = call_api(target_datetime)
-
     NED_data = format_data(json_data, logger)
 
     return NED_data.to_list()

web/package.json

@@ -56,17 +56,17 @@
     "@capacitor/app": "^6.0.1",
     "@capacitor/core": "^6.1.2",
     "@capacitor/share": "^6.0.2",
-    "@radix-ui/react-accordion": "^1.2.0",
-    "@radix-ui/react-dialog": "^1.1.1",
-    "@radix-ui/react-dropdown-menu": "^2.1.1",
-    "@radix-ui/react-navigation-menu": "^1.2.0",
-    "@radix-ui/react-portal": "^1.1.1",
-    "@radix-ui/react-slider": "^1.2.0",
-    "@radix-ui/react-switch": "^1.1.0",
-    "@radix-ui/react-toast": "^1.2.1",
+    "@radix-ui/react-accordion": "^1.2.1",
+    "@radix-ui/react-dialog": "^1.1.2",
+    "@radix-ui/react-dropdown-menu": "^2.1.2",
+    "@radix-ui/react-navigation-menu": "^1.2.1",
+    "@radix-ui/react-portal": "^1.1.2",
+    "@radix-ui/react-slider": "^1.2.1",
+    "@radix-ui/react-switch": "^1.1.1",
+    "@radix-ui/react-toast": "^1.2.2",
     "@radix-ui/react-toggle": "^1.1.0",
     "@radix-ui/react-toggle-group": "^1.1.0",
-    "@radix-ui/react-tooltip": "^1.1.2",
+    "@radix-ui/react-tooltip": "^1.1.3",
     "@react-spring/web": "^9.7.4",
     "@sentry/react": "^8.36.0",
     "@tailwindcss/forms": "0.5.3",
@@ -87,7 +87,7 @@
     "d3-shape": "^3.2.0",
     "date-fns": "^3.6.0",
     "geojson": "^0.5.0",
-    "i18next": "^23.15.1",
+    "i18next": "^23.16.4",
     "i18next-browser-languagedetector": "^8.0.0",
     "i18next-resources-to-backend": "^1.2.1",
     "jotai": "^2.9.3",
@@ -97,7 +97,7 @@
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "react-helmet-async": "^2.0.5",
-    "react-i18next": "^15.0.2",
+    "react-i18next": "^15.1.0",
     "react-icons": "^5.3.0",
     "react-map-gl": "~7.1.7",
     "react-router-dom": "6.26.2",
@@ -198,7 +198,7 @@
     "vite-plugin-pwa": "0.20.2",
     "vite-tsconfig-paths": "5.0.1",
     "vitest": "2.1.1",
-    "workbox-build": "7.1.1",
+    "workbox-build": "7.3.0",
     "workbox-window": "7.1.0"
   },
   "engines": {