Cleaned up code.

PyPSA · Oct 22, 2024 · 2aefa9b · 2aefa9b
1 parent b2c21f7
commit 2aefa9b
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 109 deletions.
diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
@@ -757,12 +757,8 @@ if config["electricity"]["base_network"] == "osm-raw":
                 "data/osm-raw/{country}/lines_way.json",
                 country=config_provider("countries"),
             ),
-            lines_cables_relation=expand(
-                "data/osm-raw/{country}/lines_cables_relation.json",
-                country=config_provider("countries"),
-            ),
-            links_relation=expand(
-                "data/osm-raw/{country}/links_relation.json",
+            routes_relation=expand(
+                "data/osm-raw/{country}/routes_relation.json",
                 country=config_provider("countries"),
             ),
             substations_way=expand(

diff --git a/rules/retrieve.smk b/rules/retrieve.smk
@@ -545,6 +545,7 @@ if config["enable"]["retrieve"] and (
         0.2: "13342577",
         0.3: "13358976",
         0.4: "13759222",
+        0.5: "13759222", # TODO update link after validation
     }
 
     # update rule to use the correct version
@@ -592,8 +593,7 @@ if config["enable"]["retrieve"] and (
         output:
             cables_way="data/osm-raw/{country}/cables_way.json",
             lines_way="data/osm-raw/{country}/lines_way.json",
-            lines_cables_relation="data/osm-raw/{country}/lines_cables_relation.json",
-            links_relation="data/osm-raw/{country}/links_relation.json",
+            routes_relation="data/osm-raw/{country}/routes_relation.json",
             substations_way="data/osm-raw/{country}/substations_way.json",
             substations_relation="data/osm-raw/{country}/substations_relation.json",
         log:
@@ -620,7 +620,7 @@ if config["enable"]["retrieve"] and (
                 country=config_provider("countries"),
             ),
             expand(
-                "data/osm-raw/{country}/links_relation.json",
+                "data/osm-raw/{country}/routes_relation.json",
                 country=config_provider("countries"),
             ),
             expand(

diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur and PyPSA-Earth Authors
 #
 # SPDX-License-Identifier: MIT
 

diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py
@@ -462,7 +462,7 @@ def _import_lines_and_cables(path_lines):
     return df_lines
 
 
-def _import_lines_cables_relation(path_relation):
+def _import_routes_relation(path_relation):
     """
 
     """
@@ -479,7 +479,7 @@ def _import_lines_cables_relation(path_relation):
     ]
     df_relation = pd.DataFrame(columns=columns)
 
-    logger.info("Importing lines/cables relation")
+    logger.info("Importing power route relations (lines, cables, links)")
     for key in path_relation:
         logger.info(f"Processing {key}...")
         for idx, ip in enumerate(path_relation[key]):
@@ -504,6 +504,7 @@ def _import_lines_cables_relation(path_relation):
                     "cables",
                     "frequency",
                     "voltage",
+                    "rating",
                 ]
 
                 tags = pd.json_normalize(df["tags"]).map(
@@ -530,89 +531,6 @@ def _import_lines_cables_relation(path_relation):
     return df_relation
 
 
-def _import_links(path_links):
-    """
-    Import links from the given input paths.
-
-    Parameters:
-    - path_links (dict): A dictionary containing the input paths for links.
-
-    Returns:
-    - df_links (DataFrame): A DataFrame containing the imported links data.
-    """
-    columns = [
-        "id",
-        "bounds",
-        "nodes",
-        "geometry",
-        "country",
-        "circuits",
-        "frequency",
-        "rating",
-        "voltage",
-    ]
-    df_links = pd.DataFrame(columns=columns)
-
-    logger.info("Importing links")
-    for key in path_links:
-        logger.info(f"Processing {key}...")
-        for idx, ip in enumerate(path_links[key]):
-            if (
-                os.path.exists(ip) and os.path.getsize(ip) > 400
-            ):  # unpopulated OSM json is about 51 bytes
-                country = os.path.basename(os.path.dirname(path_links[key][idx]))
-
-                logger.info(
-                    f" - Importing {key} {str(idx+1).zfill(2)}/{str(len(path_links[key])).zfill(2)}: {ip}"
-                )
-                with open(ip, "r") as f:
-                    data = json.load(f)
-
-                df = pd.DataFrame(data["elements"])
-                df["id"] = df["id"].astype(str)
-                df["id"] = df["id"].apply(lambda x: (f"relation/{x}"))
-                df["country"] = country
-
-                col_tags = [
-                    "circuits",
-                    "frequency",
-                    "rating",
-                    "voltage",
-                ]
-
-                tags = pd.json_normalize(df["tags"]).map(
-                    lambda x: str(x) if pd.notnull(x) else x
-                )
-
-                for ct in col_tags:
-                    if ct not in tags.columns:
-                        tags[ct] = pd.NA
-
-                tags = tags.loc[:, col_tags]
-
-                df = pd.concat([df, tags], axis="columns")
-                df.drop(columns=["type", "tags"], inplace=True)
-
-                df_links = pd.concat([df_links, df], axis="rows")
-
-            else:
-                logger.info(
-                    f" - Skipping {key} {str(idx+1).zfill(2)}/{str(len(path_links[key])).zfill(2)} (empty): {ip}"
-                )
-                continue
-        logger.info("---")
-        logger.info("Dropping lines without rating.")
-        len_before = len(df_links)
-        df_links = df_links.dropna(subset=["rating"])
-        len_after = len(df_links)
-        logger.info(
-            f"Dropped {len_before-len_after} elements without rating. "
-            + f"Imported {len_after} elements."
-        )
-
-    return df_links
-
-
 def _create_single_link(row):
     """
     Create a single link from multiple rows within a OSM link relation.
@@ -1687,11 +1605,13 @@ def _check_if_ways_in_multi(list, longer_list):
     ### Lines/Cables relations
     logger.info("---")
     logger.info("AC LINES/CABLES RELATIONS")
-    path_lines_cables_relation = {
-        "lines_cables_relation": snakemake.input.lines_cables_relation,
+    path_routes_relation = {
+        "routes_relation": snakemake.input.routes_relation,
     }
 
-    df_lines_cables_relation = _import_lines_cables_relation(path_lines_cables_relation)
+    df_routes_relation = _import_routes_relation(path_routes_relation)
+
+    df_lines_cables_relation = df_routes_relation.copy()
     df_lines_cables_relation = _drop_duplicate_lines(df_lines_cables_relation)
     df_lines_cables_relation.loc[:, "voltage"] = _clean_voltage(df_lines_cables_relation["voltage"])
     df_lines_cables_relation, list_voltages = _filter_by_voltage(df_lines_cables_relation, min_voltage=min_voltage_ac)
@@ -1787,11 +1707,17 @@ def _check_if_ways_in_multi(list, longer_list):
 
     logger.info("---")
     logger.info("HVDC LINKS")
-    path_links = {
-        "links": snakemake.input.links_relation,
-    }
 
-    df_links = _import_links(path_links)
+    df_links = df_routes_relation.copy()
+
+    logger.info("Dropping lines without rating.")
+    len_before = len(df_links)
+    df_links = df_links.dropna(subset=["rating"])
+    len_after = len(df_links)
+    logger.info(
+        f"Dropped {len_before-len_after} elements without rating. "
+        + f"Imported {len_after} elements."
+    )
 
     df_links = _drop_duplicate_lines(df_links)
     df_links.loc[:, "voltage"] = _clean_voltage(df_links["voltage"])

diff --git a/scripts/prepare_osm_network_release.py b/scripts/prepare_osm_network_release.py
@@ -19,6 +19,7 @@
     "dc",
     "symbol",
     "under_construction",
+    "tags",
     "x",
     "y",
     "country",
@@ -35,7 +36,7 @@
     "under_construction",
     "geometry",
 ]
-LINKS_COLUMNS = [
+LINKS_COLUMNS = [ 
     "link_id",
     "bus0",
     "bus1",
@@ -44,6 +45,7 @@
     "length",
     "underground",
     "under_construction",
+    "tags",
     "geometry",
 ]
 TRANSFORMERS_COLUMNS = [
@@ -52,13 +54,15 @@
     "bus1",
     "voltage_bus0",
     "voltage_bus1",
+    "s_nom",
     "geometry",
 ]
 CONVERTERS_COLUMNS = [
     "converter_id",
     "bus0",
     "bus1",
     "voltage",
+    "p_nom",
     "geometry",
 ]
 

diff --git a/scripts/retrieve_osm_data.py b/scripts/retrieve_osm_data.py
@@ -31,8 +31,7 @@ def retrieve_osm_data(
     features=[
         "cables_way",
         "lines_way",
-        "lines_cables_relation",
-        "links_relation",
+        "routes_relation",
         "substations_way",
         "substations_relation",
     ],
@@ -52,8 +51,7 @@ def retrieve_osm_data(
         A list of OSM features to retrieve. The default is [
             "cables_way",
             "lines_way",
-            "lines_cables_relation",
-            "links_relation",
+            "routes_relation",
             "substations_way",
             "substations_relation",
             ].
@@ -64,8 +62,7 @@ def retrieve_osm_data(
     features_dict = {
         "cables_way": 'way["power"="cable"]',
         "lines_way": 'way["power"="line"]',
-        "lines_cables_relation": 'relation["route"="power"]',
-        "links_relation": 'relation["route"="power"]["frequency"="0"]',
+        "routes_relation": 'relation["route"="power"]',
         "substations_way": 'way["power"="substation"]',
         "substations_relation": 'relation["power"="substation"]',
     }