diff --git a/README.md b/README.md index 8ecbfd951..bb10b50a7 100644 --- a/README.md +++ b/README.md @@ -90,4 +90,4 @@ To generate a PDF of the dependency graph of all steps `build/dag.pdf` run: ## License -The code in this repo is MIT licensed, see `./LICENSE.md`. \ No newline at end of file +The code in this repo is MIT licensed, see `./LICENSE.md`. diff --git a/rules/common.smk b/rules/common.smk index 332103f26..52d25964b 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -7,10 +7,9 @@ from functools import partial, lru_cache import os, sys, glob -path = workflow.source_path("../scripts/_helpers.py") -sys.path.insert(0, os.path.dirname(path)) +sys.path.insert(0, os.path.abspath("../scripts")) -from _helpers import validate_checksum, update_config_from_wildcards +from scripts._helpers import validate_checksum, update_config_from_wildcards from snakemake.utils import update_config diff --git a/scripts/_benchmark.py b/scripts/_benchmark.py index 58fc3d399..5fc4b21ee 100644 --- a/scripts/_benchmark.py +++ b/scripts/_benchmark.py @@ -2,9 +2,7 @@ # SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT -""" - -""" +""" """ from __future__ import absolute_import, print_function diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index ff5e91515..4ce7c4497 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -12,14 +12,15 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import ( +from add_existing_baseyear import add_build_year_to_new_assets +from pypsa.clustering.spatial import normed_or_uniform + +from scripts._helpers import ( configure_logging, get_snapshots, set_scenario_config, update_config_from_wildcards, ) -from add_existing_baseyear import add_build_year_to_new_assets -from pypsa.clustering.spatial import normed_or_uniform logger = logging.getLogger(__name__) idx = pd.IndexSlice @@ -258,7 +259,7 @@ def update_heat_pump_efficiency(n: pypsa.Network, n_p: pypsa.Network, year: int) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "add_brownfield", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 215ae47d2..85d260418 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -119,14 +119,15 @@ import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import ( +from powerplantmatching.export import map_country_bus +from pypsa.clustering.spatial import DEFAULT_ONE_PORT_STRATEGIES, normed_or_uniform + +from scripts._helpers import ( configure_logging, get_snapshots, set_scenario_config, update_p_nom_max, ) -from powerplantmatching.export import map_country_bus -from pypsa.clustering.spatial import DEFAULT_ONE_PORT_STRATEGIES, normed_or_uniform idx = pd.IndexSlice @@ -308,7 +309,6 @@ def load_and_aggregate_powerplants( aggregation_strategies: dict = None, exclude_carriers: list = None, ) -> pd.DataFrame: - if not aggregation_strategies: aggregation_strategies = {} @@ -410,7 +410,6 @@ def attach_load( busmap_fn: str, scaling: float = 1.0, ) -> None: - load = ( xr.open_dataarray(load_fn).to_dataframe().squeeze(axis=1).unstack(level="time") ) @@ -431,7 +430,6 @@ def set_transmission_costs( line_length_factor: float = 1.0, link_length_factor: float = 1.0, ) -> None: - n.lines["capital_cost"] = ( n.lines["length"] * line_length_factor @@ -1014,7 +1012,7 @@ def attach_stores(n, costs, extendable_carriers): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("add_electricity", clusters=100) configure_logging(snakemake) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 3136ae4f9..53ef61dec 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -18,16 +18,21 @@ import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import ( + +from scripts._helpers import ( configure_logging, set_scenario_config, update_config_from_wildcards, ) -from add_electricity import sanitize_carriers -from definitions.heat_sector import HeatSector -from definitions.heat_system import HeatSystem -from definitions.heat_system_type import HeatSystemType -from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs +from scripts.add_electricity import sanitize_carriers +from scripts.definitions.heat_sector import HeatSector +from scripts.definitions.heat_system import HeatSystem +from scripts.definitions.heat_system_type import HeatSystemType +from scripts.prepare_sector_network import ( + cluster_heat_buses, + define_spatial, + prepare_costs, +) logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -1063,7 +1068,7 @@ def set_defaults(n): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "add_existing_baseyear", diff --git a/scripts/add_transmission_projects_and_dlr.py b/scripts/add_transmission_projects_and_dlr.py index 88cbfb59b..75e372c9f 100644 --- a/scripts/add_transmission_projects_and_dlr.py +++ b/scripts/add_transmission_projects_and_dlr.py @@ -13,7 +13,8 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ def attach_line_rating( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("add_transmission_projects_and_dlr") configure_logging(snakemake) @@ -82,11 +83,9 @@ def attach_line_rating( n = pypsa.Network(snakemake.input.network) if params["transmission_projects"]["enable"]: - attach_transmission_projects(n, snakemake.input.transmission_projects) if params["dlr"]["activate"]: - rating = xr.open_dataarray(snakemake.input.dlr).to_pandas().transpose() s_max_pu = params["s_max_pu"] diff --git a/scripts/base_network.py b/scripts/base_network.py index 71eefc5c3..bac24e17b 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -86,14 +86,19 @@ import shapely.prepared import shapely.wkt import yaml -from _helpers import REGION_COLS, configure_logging, get_snapshots, set_scenario_config from packaging.version import Version, parse from scipy.sparse import csgraph from scipy.spatial import KDTree from shapely.geometry import LineString, Point -PD_GE_2_2 = parse(pd.__version__) >= Version("2.2") +from scripts._helpers import ( + REGION_COLS, + configure_logging, + get_snapshots, + set_scenario_config, +) +PD_GE_2_2 = parse(pd.__version__) >= Version("2.2") logger = logging.getLogger(__name__) @@ -976,7 +981,7 @@ def append_bus_shapes(n, shapes, type): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("base_network") configure_logging(snakemake) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 0c6f85d2f..8134607f5 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -25,14 +25,15 @@ import country_converter as coco import numpy as np import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config cc = coco.CountryConverter() if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_ammonia_production") diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 32402eb25..1599cee02 100755 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -12,8 +12,9 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, set_scenario_config -from build_energy_totals import build_eurostat + +from scripts._helpers import configure_logging, set_scenario_config +from scripts.build_energy_totals import build_eurostat logger = logging.getLogger(__name__) AVAILABLE_BIOMASS_YEARS = [2010, 2020, 2030, 2040, 2050] @@ -339,8 +340,7 @@ def add_unsustainable_potentials(df): if __name__ == "__main__": if "snakemake" not in globals(): - - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_biomass_potentials", diff --git a/scripts/build_biomass_transport_costs.py b/scripts/build_biomass_transport_costs.py index d18362306..86e4214aa 100644 --- a/scripts/build_biomass_transport_costs.py +++ b/scripts/build_biomass_transport_costs.py @@ -16,6 +16,7 @@ @author: bw0928 """ + import pandas as pd ENERGY_CONTENT = 4.8 # unit MWh/t (wood pellets) @@ -114,7 +115,7 @@ def build_biomass_transport_costs(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_biomass_transport_costs") diff --git a/scripts/build_central_heating_temperature_profiles/run.py b/scripts/build_central_heating_temperature_profiles/run.py index c04c96b55..1b448d960 100644 --- a/scripts/build_central_heating_temperature_profiles/run.py +++ b/scripts/build_central_heating_temperature_profiles/run.py @@ -41,11 +41,12 @@ import numpy as np import pandas as pd import xarray as xr -from _helpers import set_scenario_config from central_heating_temperature_approximator import ( CentralHeatingTemperatureApproximator, ) +from scripts._helpers import set_scenario_config + def extrapolate_missing_supply_temperatures_by_country( extrapolate_from: dict, extrapolate_to: dict @@ -187,7 +188,7 @@ def scale_temperature_to_investment_year( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_cop_profiles", diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 013059529..4f11ecc1b 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -11,11 +11,12 @@ import geopandas as gpd import pandas as pd import xarray as xr -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_clustered_population_layouts", clusters=48) diff --git a/scripts/build_cop_profiles/BaseCopApproximator.py b/scripts/build_cop_profiles/BaseCopApproximator.py index 10019b3d3..8d2832d4e 100644 --- a/scripts/build_cop_profiles/BaseCopApproximator.py +++ b/scripts/build_cop_profiles/BaseCopApproximator.py @@ -65,7 +65,7 @@ def approximate_cop(self) -> Union[xr.DataArray, np.array]: @staticmethod def celsius_to_kelvin( - t_celsius: Union[float, xr.DataArray, np.array] + t_celsius: Union[float, xr.DataArray, np.array], ) -> Union[float, xr.DataArray, np.array]: """ Convert temperature from Celsius to Kelvin. diff --git a/scripts/build_cop_profiles/run.py b/scripts/build_cop_profiles/run.py index b93f7df69..a4a3e2eb6 100644 --- a/scripts/build_cop_profiles/run.py +++ b/scripts/build_cop_profiles/run.py @@ -46,10 +46,10 @@ import numpy as np import pandas as pd import xarray as xr -from _helpers import set_scenario_config from CentralHeatingCopApproximator import CentralHeatingCopApproximator from DecentralHeatingCopApproximator import DecentralHeatingCopApproximator +from scripts._helpers import set_scenario_config from scripts.definitions.heat_system_type import HeatSystemType @@ -100,7 +100,7 @@ def get_country_from_node_name(node_name: str) -> str: if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_cop_profiles", diff --git a/scripts/build_cross_border_flows.py b/scripts/build_cross_border_flows.py index d463d234f..ae5d6811b 100644 --- a/scripts/build_cross_border_flows.py +++ b/scripts/build_cross_border_flows.py @@ -8,16 +8,17 @@ import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config from entsoe import EntsoePandasClient from entsoe.exceptions import InvalidBusinessParameterError, NoMatchingDataError from requests import HTTPError +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_cross_border_flows") configure_logging(snakemake) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 015eb66a8..cad146389 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -95,13 +95,14 @@ import atlite import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_cutout", cutout="europe-2013-sarah3-era5") configure_logging(snakemake) diff --git a/scripts/build_daily_heat_demand.py b/scripts/build_daily_heat_demand.py index 6bcb6bd8a..c96607599 100644 --- a/scripts/build_daily_heat_demand.py +++ b/scripts/build_daily_heat_demand.py @@ -48,12 +48,13 @@ import geopandas as gpd import numpy as np import xarray as xr -from _helpers import get_snapshots, set_scenario_config from dask.distributed import Client, LocalCluster +from scripts._helpers import get_snapshots, set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_daily_heat_demands", diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index e84819189..ac4af9bdb 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -32,15 +32,16 @@ import logging import pandas as pd -from _helpers import configure_logging, set_scenario_config -from prepare_sector_network import get + +from scripts._helpers import configure_logging, set_scenario_config +from scripts.prepare_sector_network import get logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_district_heat_share", diff --git a/scripts/build_egs_potentials.py b/scripts/build_egs_potentials.py index d80238695..05529f6d8 100644 --- a/scripts/build_egs_potentials.py +++ b/scripts/build_egs_potentials.py @@ -107,11 +107,9 @@ def prepare_capex(prepared_data): capex_df = pd.DataFrame(columns=prepared_data.keys()) for year in capex_df.columns: - year_data = prepared_data[year].groupby("geometry").mean().reset_index() for g in year_data.geometry: - if not g in year_data.geometry.tolist(): # weird but apparently necessary continue @@ -124,7 +122,6 @@ def prepare_capex(prepared_data): # fill up missing values assuming cost reduction factors similar to existing values for sooner, later in zip(capex_df.columns[::-1][1:], capex_df.columns[::-1]): - missing_mask = capex_df[sooner].isna() cr_factor = ( capex_df.loc[~missing_mask, later] / capex_df.loc[~missing_mask, sooner] @@ -197,7 +194,7 @@ def get_capacity_factors(network_regions_file, air_temperatures_file): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_egs_potentials", diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index d3334ec85..f2d01a1cb 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -39,9 +39,10 @@ import numpy as np import pandas as pd -from _helpers import configure_logging, get_snapshots, set_scenario_config from pandas import Timedelta as Delta +from scripts._helpers import configure_logging, get_snapshots, set_scenario_config + logger = logging.getLogger(__name__) @@ -256,7 +257,7 @@ def manual_adjustment(load, fn_load, countries): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_electricity_demand") diff --git a/scripts/build_electricity_demand_base.py b/scripts/build_electricity_demand_base.py index f48cc1a48..90a29c907 100644 --- a/scripts/build_electricity_demand_base.py +++ b/scripts/build_electricity_demand_base.py @@ -15,9 +15,10 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, set_scenario_config from shapely.prepared import prep +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -60,7 +61,6 @@ def upsample_load( data_arrays = [] for cntry, group in gdf_regions.geometry.groupby(gdf_regions.country): - load_ct = load[cntry] if cntry in ["UA", "MD"]: @@ -103,7 +103,7 @@ def upsample_load( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_electricity_demand_base") configure_logging(snakemake) diff --git a/scripts/build_electricity_prices.py b/scripts/build_electricity_prices.py index f9b964bd0..18b650ec0 100644 --- a/scripts/build_electricity_prices.py +++ b/scripts/build_electricity_prices.py @@ -7,15 +7,16 @@ import logging import pandas as pd -from _helpers import configure_logging, set_scenario_config from entsoe import EntsoePandasClient from entsoe.exceptions import NoMatchingDataError +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_cross_border_flows") configure_logging(snakemake) diff --git a/scripts/build_electricity_production.py b/scripts/build_electricity_production.py index b81c6b45a..57f5a7bf8 100644 --- a/scripts/build_electricity_production.py +++ b/scripts/build_electricity_production.py @@ -7,10 +7,11 @@ import logging import pandas as pd -from _helpers import configure_logging, set_scenario_config from entsoe import EntsoePandasClient from entsoe.exceptions import NoMatchingDataError +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -35,7 +36,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_electricity_production") configure_logging(snakemake) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 6afa6b334..1df27ef07 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -47,9 +47,10 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, mute_print, set_scenario_config from tqdm import tqdm +from scripts._helpers import configure_logging, mute_print, set_scenario_config + cc = coco.CountryConverter() logger = logging.getLogger(__name__) idx = pd.IndexSlice @@ -1394,7 +1395,6 @@ def rescale_idees_from_eurostat( filling_years = [(2015, slice(2016, 2021)), (2000, slice(1990, 1999))] for source_year, target_years in filling_years: - slicer_source = idx[country, source_year, :, :] slicer_target = idx[country, target_years, :, :] @@ -1493,7 +1493,6 @@ def update_residential_from_eurostat(energy: pd.DataFrame) -> pd.DataFrame: } for nrg_name, (code, siec) in nrg_type.items(): - # Select energy balance type, rename columns and countries to match IDEES data, # convert TJ to TWh col_to_rename = {"geo": "country", "TIME_PERIOD": "year", "OBS_VALUE": nrg_name} @@ -1597,7 +1596,7 @@ def build_heating_efficiencies( # %% if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_energy_totals") diff --git a/scripts/build_existing_heating_distribution.py b/scripts/build_existing_heating_distribution.py index 280375cea..8467dbe17 100644 --- a/scripts/build_existing_heating_distribution.py +++ b/scripts/build_existing_heating_distribution.py @@ -38,10 +38,12 @@ ----------- - "Mapping and analyses of the current and future (2020 - 2030) heating/cooling fuel deployment (fossil/renewables)" (https://energy.ec.europa.eu/publications/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment-fossilrenewables-1_en) """ + import country_converter as coco import numpy as np import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config cc = coco.CountryConverter() @@ -156,7 +158,7 @@ def build_existing_heating(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_existing_heating_distribution", diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 07ba89718..0c1216643 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -12,9 +12,10 @@ import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config from cluster_gas_network import load_bus_regions +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -137,7 +138,7 @@ def build_gas_input_locations(gem_fn, entry_fn, sto_fn, countries): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_gas_input_locations", diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index e16096d36..7be448404 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -12,10 +12,11 @@ import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config from pypsa.geo import haversine_pts from shapely.geometry import Point +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -142,7 +143,7 @@ def prepare_dataset( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_gas_network") diff --git a/scripts/build_gdp_pop_non_nuts3.py b/scripts/build_gdp_pop_non_nuts3.py index cd3f69f3b..72b002bc1 100644 --- a/scripts/build_gdp_pop_non_nuts3.py +++ b/scripts/build_gdp_pop_non_nuts3.py @@ -18,10 +18,11 @@ import pypsa import rasterio import xarray as xr -from _helpers import configure_logging, set_scenario_config from rasterio.mask import mask from shapely.geometry import box +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -121,7 +122,7 @@ def calc_gdp_pop(country, regions, gdp_non_nuts3, pop_non_nuts3): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_gdp_pop_non_nuts3") configure_logging(snakemake) diff --git a/scripts/build_hac_features.py b/scripts/build_hac_features.py index 08d5e1201..d483da1bc 100644 --- a/scripts/build_hac_features.py +++ b/scripts/build_hac_features.py @@ -10,15 +10,16 @@ import atlite import geopandas as gpd -from _helpers import get_snapshots, set_scenario_config from atlite.aggregate import aggregate_matrix from dask.distributed import Client +from scripts._helpers import get_snapshots, set_scenario_config + logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_hac_features") set_scenario_config(snakemake) diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py index 7a43c9ca9..6575986f3 100644 --- a/scripts/build_heat_totals.py +++ b/scripts/build_heat_totals.py @@ -101,7 +101,7 @@ def approximate_heat_demand(energy_totals: pd.DataFrame, hdd: pd.DataFrame): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_heat_totals") diff --git a/scripts/build_hourly_heat_demand.py b/scripts/build_hourly_heat_demand.py index 9bb1f77ff..fcb2d4273 100644 --- a/scripts/build_hourly_heat_demand.py +++ b/scripts/build_hourly_heat_demand.py @@ -34,12 +34,16 @@ import pandas as pd import xarray as xr -from _helpers import generate_periodic_profiles, get_snapshots, set_scenario_config + +from scripts._helpers import ( + generate_periodic_profiles, + get_snapshots, + mock_snakemake, + set_scenario_config, +) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake - snakemake = mock_snakemake( "build_hourly_heat_demand", scope="total", diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 2d0d2e521..d869d236c 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,9 +65,10 @@ import country_converter as coco import geopandas as gpd import pandas as pd -from _helpers import configure_logging, get_snapshots, set_scenario_config from numpy.polynomial import Polynomial +from scripts._helpers import configure_logging, get_snapshots, set_scenario_config + cc = coco.CountryConverter() @@ -170,7 +171,7 @@ def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 8fe707a41..e9544d45b 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -43,7 +43,8 @@ import country_converter as coco import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -384,7 +385,7 @@ def build_nodal_distribution_key( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_distribution_key", diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index f011e61f6..4d181b3b0 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -65,9 +65,10 @@ import country_converter as coco import pandas as pd -from _helpers import set_scenario_config from tqdm import tqdm +from scripts._helpers import set_scenario_config + cc = coco.CountryConverter() ktoe_to_twh = 0.011630 @@ -283,7 +284,7 @@ def add_coke_ovens(demand, fn, year, factor=0.75): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") set_scenario_config(snakemake) diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py index db9cac6b4..7f430398e 100644 --- a/scripts/build_industrial_energy_demand_per_node.py +++ b/scripts/build_industrial_energy_demand_per_node.py @@ -37,11 +37,12 @@ """ import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_energy_demand_per_node", diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py index 0675ce079..3b3b0b7e7 100644 --- a/scripts/build_industrial_energy_demand_per_node_today.py +++ b/scripts/build_industrial_energy_demand_per_node_today.py @@ -29,7 +29,8 @@ import numpy as np import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config # map JRC/our sectors to hotmaps sector, where mapping exist sector_mapping = { @@ -88,7 +89,7 @@ def build_nodal_industrial_energy_demand(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_energy_demand_per_node_today", diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 63f4c1c15..786fcdf17 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -68,9 +68,10 @@ import country_converter as coco import numpy as np import pandas as pd -from _helpers import configure_logging, mute_print, set_scenario_config from tqdm import tqdm +from scripts._helpers import configure_logging, mute_print, set_scenario_config + logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -333,7 +334,7 @@ def separate_basic_chemicals(demand, year): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_country") configure_logging(snakemake) diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 9bcb7e5a4..aff85d40c 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -60,12 +60,13 @@ """ import pandas as pd -from _helpers import set_scenario_config -from prepare_sector_network import get + +from scripts._helpers import set_scenario_config +from scripts.prepare_sector_network import get if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") set_scenario_config(snakemake) diff --git a/scripts/build_industrial_production_per_node.py b/scripts/build_industrial_production_per_node.py index cdc6e1a20..fd837f8fb 100644 --- a/scripts/build_industrial_production_per_node.py +++ b/scripts/build_industrial_production_per_node.py @@ -28,7 +28,8 @@ from itertools import product import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config # map JRC/our sectors to hotmaps sector, where mapping exist sector_mapping = { @@ -85,7 +86,7 @@ def build_nodal_industrial_production(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_node", clusters=48) set_scenario_config(snakemake) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index cd27a3a46..74587c625 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -52,7 +52,8 @@ import country_converter as coco import pandas as pd -from _helpers import mute_print, set_scenario_config + +from scripts._helpers import mute_print, set_scenario_config logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -1574,7 +1575,6 @@ def wood_and_wood_products(): def other_industrial_sectors(): - sector = "Other industrial sectors" idees = load_idees_data(sector) @@ -1646,7 +1646,7 @@ def other_industrial_sectors(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_industry_sector_ratios") set_scenario_config(snakemake) diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py index e47ad0b7f..e7af081c8 100644 --- a/scripts/build_industry_sector_ratios_intermediate.py +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -81,11 +81,11 @@ import numpy as np import pandas as pd -from prepare_sector_network import get +from scripts.prepare_sector_network import get -def build_industry_sector_ratios_intermediate(): +def build_industry_sector_ratios_intermediate(): # in TWh/a demand = pd.read_csv( snakemake.input.industrial_energy_demand_per_country_today, @@ -143,7 +143,7 @@ def build_industry_sector_ratios_intermediate(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_industry_sector_ratios_intermediate", diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 86e98d62b..948af1fb0 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -59,11 +59,12 @@ import numpy as np import pypsa import xarray as xr -from _helpers import configure_logging, get_snapshots, set_scenario_config from dask.distributed import Client from shapely.geometry import LineString as Line from shapely.geometry import Point +from scripts._helpers import configure_logging, get_snapshots, set_scenario_config + logger = logging.getLogger(__name__) @@ -155,7 +156,7 @@ def calculate_line_rating( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_line_rating") configure_logging(snakemake) diff --git a/scripts/build_monthly_prices.py b/scripts/build_monthly_prices.py index d35243c34..0320adea5 100644 --- a/scripts/build_monthly_prices.py +++ b/scripts/build_monthly_prices.py @@ -43,7 +43,8 @@ import logging import pandas as pd -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -106,7 +107,7 @@ def get_co2_price(): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_monthly_prices") diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 83461a98d..9122d4014 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -10,11 +10,12 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, set_scenario_config from shapely.geometry import LineString, Point from shapely.ops import linemerge, nearest_points, split from tqdm import tqdm +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -761,7 +762,6 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): def build_network(inputs, outputs): - logger.info("Reading input data.") buses = gpd.read_file(inputs["substations"]) lines = gpd.read_file(inputs["lines"]) @@ -851,7 +851,7 @@ def build_network(inputs, outputs): if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_osm_network") diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index b9af8d850..9c22ce6db 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -14,7 +14,8 @@ import numpy as np import pandas as pd import xarray as xr -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_population_layouts", diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 74c2573bf..e27d827f1 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -7,11 +7,12 @@ """ import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_population_weighted_energy_totals", diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 94d03585b..4c837c38f 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -91,9 +91,10 @@ import pandas as pd import powerplantmatching as pm import pypsa -from _helpers import configure_logging, set_scenario_config from powerplantmatching.export import map_country_bus +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -165,7 +166,7 @@ def replace_natural_gas_fueltype(df): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_powerplants") configure_logging(snakemake) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index ab6a4748d..748a6bf81 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -118,22 +118,24 @@ The maximal installable potential for the node (`p_nom_max`) is computed by adding up the installable potentials of the individual grid cells. """ + import logging import time import atlite import geopandas as gpd import xarray as xr -from _helpers import configure_logging, get_snapshots, set_scenario_config from build_shapes import _simplify_polys from dask.distributed import Client +from scripts._helpers import configure_logging, get_snapshots, set_scenario_config + logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_renewable_profiles", clusters=38, technology="offwind-ac" @@ -214,7 +216,6 @@ profiles = [] for year, model in models.items(): - logger.info( f"Calculate weighted capacity factor time series for model {model} for technology {technology}..." ) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index b25b6a648..6a0b71808 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -66,9 +66,11 @@ (4) get cost savings per retrofitting measures for each sector by weighting with heated floor area """ + import pandas as pd import xarray as xr -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config # (i) --- FIXED PARAMETER / STANDARD VALUES ----------------------------------- @@ -1046,7 +1048,7 @@ def sample_dE_costs_area( # %% --- MAIN -------------------------------------------------------------- if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_retro_cost", diff --git a/scripts/build_salt_cavern_potentials.py b/scripts/build_salt_cavern_potentials.py index 2923da16b..823c4a10c 100644 --- a/scripts/build_salt_cavern_potentials.py +++ b/scripts/build_salt_cavern_potentials.py @@ -21,10 +21,10 @@ onshore (>50km from sea), offshore (Figure 7). """ - import geopandas as gpd import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config def concat_gdf(gdf_list, crs="EPSG:4326"): @@ -72,7 +72,7 @@ def salt_cavern_potential_by_region(caverns, regions): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_salt_cavern_potentials", clusters="37") diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index a153a282c..0e1347d8f 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -10,7 +10,8 @@ import geopandas as gpd import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config def area(gdf): @@ -36,7 +37,7 @@ def allocate_sequestration_potential( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_sequestration_potentials", clusters="128") diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 29eb91478..c757b0cfb 100755 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -77,9 +77,10 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, set_scenario_config from shapely.geometry import MultiPolygon, Polygon +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -106,7 +107,6 @@ def _simplify_polys(polys, minarea=0.1, tolerance=None, filterremote=True): def countries(naturalearth, country_list): - df = gpd.read_file(naturalearth) # Names are a hassle in naturalearth, try several fields @@ -242,7 +242,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_shapes") configure_logging(snakemake) diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py index 12befe995..a0aab9dac 100644 --- a/scripts/build_ship_raster.py +++ b/scripts/build_ship_raster.py @@ -47,7 +47,8 @@ import atlite import rioxarray -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -79,7 +80,7 @@ def determine_cutout_xXyY(cutout_name): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_ship_raster") configure_logging(snakemake) diff --git a/scripts/build_shipping_demand.py b/scripts/build_shipping_demand.py index 649e53c7b..49bc3998a 100644 --- a/scripts/build_shipping_demand.py +++ b/scripts/build_shipping_demand.py @@ -11,11 +11,12 @@ import geopandas as gpd import pandas as pd -from _helpers import set_scenario_config + +from scripts._helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_shipping_demand", clusters=48) set_scenario_config(snakemake) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 4f13b5fe1..269d0e8c3 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -39,12 +39,13 @@ import geopandas as gpd import numpy as np import xarray as xr -from _helpers import get_snapshots, set_scenario_config from dask.distributed import Client, LocalCluster +from scripts._helpers import get_snapshots, set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_solar_thermal_profiles", clusters=48) set_scenario_config(snakemake) @@ -80,7 +81,7 @@ matrix=M_tilde.T, index=clustered_regions.index, dask_kwargs=dict(scheduler=client), - show_progress=False + show_progress=False, ) solar_thermal.to_netcdf(snakemake.output.solar_thermal) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 34cd39d84..6a7cc38d6 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -40,12 +40,13 @@ import geopandas as gpd import numpy as np import xarray as xr -from _helpers import get_snapshots, set_scenario_config from dask.distributed import Client, LocalCluster +from scripts._helpers import get_snapshots, set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_temperature_profiles", diff --git a/scripts/build_transmission_projects.py b/scripts/build_transmission_projects.py index aa2c53698..0e0049d76 100644 --- a/scripts/build_transmission_projects.py +++ b/scripts/build_transmission_projects.py @@ -54,11 +54,12 @@ import pandas as pd import pypsa import shapely -from _helpers import configure_logging, set_scenario_config from pypsa.descriptors import nominal_attrs from scipy import spatial from shapely.geometry import LineString, Point +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -470,7 +471,7 @@ def fill_length_from_geometry(line, line_factor=1.2): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_transmission_projects", run="all") configure_logging(snakemake) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index f2ae0c21c..c6e477a1d 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -13,7 +13,8 @@ import numpy as np import pandas as pd import xarray as xr -from _helpers import ( + +from scripts._helpers import ( configure_logging, generate_periodic_profiles, get_snapshots, @@ -165,7 +166,7 @@ def bev_dsm_profile(snapshots, nodes, options): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_transport_demand", clusters=128) configure_logging(snakemake) diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index 8669d0af5..2065d0ec8 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -21,10 +21,11 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, set_scenario_config from shapely.geometry import LineString, MultiLineString, Point, Polygon from shapely.ops import linemerge, unary_union +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -1659,7 +1660,7 @@ def _extend_lines_to_substations(gdf_lines, gdf_substations_polygon): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("clean_osm_data") diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index 559983964..d40e9c9f8 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -10,10 +10,11 @@ import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config from pypsa.geo import haversine_pts from shapely import wkt +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -106,7 +107,7 @@ def aggregate_parallel_pipes(df): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("cluster_gas_network", clusters="37") configure_logging(snakemake) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index f760c964d..cc56bc9e9 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -104,7 +104,6 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging, set_scenario_config from base_network import append_bus_shapes from packaging.version import Version, parse from pypsa.clustering.spatial import ( @@ -115,6 +114,8 @@ ) from scipy.sparse.csgraph import connected_components +from scripts._helpers import configure_logging, set_scenario_config + PD_GE_2_2 = parse(pd.__version__) >= Version("2.2") warnings.filterwarnings(action="ignore", category=UserWarning) @@ -141,7 +142,6 @@ def get_feature_data_for_hac(fn: str) -> pd.DataFrame: def fix_country_assignment_for_hac(n: pypsa.Network) -> None: - # overwrite country of nodes that are disconnected from their country-topology for country in n.buses.country.unique(): m = n[n.buses.country == country].copy() @@ -293,7 +293,6 @@ def clustering_for_n_clusters( line_length_factor: float = 1.25, aggregation_strategies: dict | None = None, ) -> pypsa.clustering.spatial.Clustering: - if aggregation_strategies is None: aggregation_strategies = dict() @@ -340,7 +339,7 @@ def cluster_regions( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("cluster_network", clusters=60) configure_logging(snakemake) diff --git a/scripts/definitions/heat_system.py b/scripts/definitions/heat_system.py index 2806f6bf4..b907b0fef 100644 --- a/scripts/definitions/heat_system.py +++ b/scripts/definitions/heat_system.py @@ -5,8 +5,8 @@ from enum import Enum -from definitions.heat_sector import HeatSector -from definitions.heat_system_type import HeatSystemType +from scripts.definitions.heat_sector import HeatSector +from scripts.definitions.heat_system_type import HeatSystemType class HeatSystem(Enum): diff --git a/scripts/determine_availability_matrix.py b/scripts/determine_availability_matrix.py index 8006740e3..228a12bed 100644 --- a/scripts/determine_availability_matrix.py +++ b/scripts/determine_availability_matrix.py @@ -72,6 +72,7 @@ - ``resources/availability_matrix_{clusters_{technology}.nc`` """ + import functools import logging import time @@ -80,14 +81,15 @@ import geopandas as gpd import numpy as np import xarray as xr -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_renewable_profiles", clusters=100, technology="onwind" diff --git a/scripts/determine_availability_matrix_MD_UA.py b/scripts/determine_availability_matrix_MD_UA.py index 424f19373..395bf1e59 100644 --- a/scripts/determine_availability_matrix_MD_UA.py +++ b/scripts/determine_availability_matrix_MD_UA.py @@ -16,7 +16,8 @@ import fiona import geopandas as gpd import numpy as np -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -31,7 +32,7 @@ def get_wdpa_layer_name(wdpa_fn, layer_substring): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "determine_availability_matrix_MD_UA", clusters=100, technology="solar" diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 1f92b91aa..e329fe1c3 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -13,8 +13,9 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, get_snapshots, set_scenario_config -from prepare_sector_network import prepare_costs + +from scripts._helpers import configure_logging, get_snapshots, set_scenario_config +from scripts.prepare_sector_network import prepare_costs idx = pd.IndexSlice logger = logging.getLogger(__name__) @@ -751,7 +752,7 @@ def to_csv(df): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("make_summary") diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 842eb000c..975ba4910 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -8,19 +8,19 @@ other metrics. """ - import numpy as np import pandas as pd import pypsa -from _helpers import set_scenario_config from make_summary import calculate_cfs # noqa: F401 from make_summary import calculate_nodal_cfs # noqa: F401 from make_summary import calculate_nodal_costs # noqa: F401 from make_summary import assign_carriers, assign_locations -from prepare_sector_network import prepare_costs from pypsa.descriptors import get_active_assets from six import iteritems +from scripts._helpers import set_scenario_config +from scripts.prepare_sector_network import prepare_costs + idx = pd.IndexSlice opt_name = {"Store": "e", "Line": "s", "Transformer": "s"} @@ -720,7 +720,7 @@ def to_csv(df): if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("make_summary_perfect") set_scenario_config(snakemake) diff --git a/scripts/plot_gas_network.py b/scripts/plot_gas_network.py index 9098bbc0a..3409a4ee0 100644 --- a/scripts/plot_gas_network.py +++ b/scripts/plot_gas_network.py @@ -13,10 +13,11 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config -from plot_power_network import assign_location, load_projection from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches +from scripts._helpers import configure_logging, set_scenario_config +from scripts.plot_power_network import assign_location, load_projection + logger = logging.getLogger(__name__) @@ -229,7 +230,7 @@ def plot_ch4_map(n): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_gas_network", diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index 57850116b..1e0fe13d1 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -13,10 +13,11 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config -from plot_power_network import assign_location, load_projection from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches +from scripts._helpers import configure_logging, set_scenario_config +from scripts.plot_power_network import assign_location, load_projection + logger = logging.getLogger(__name__) @@ -252,7 +253,7 @@ def plot_h2_map(n, regions): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_hydrogen_network", diff --git a/scripts/plot_power_network.py b/scripts/plot_power_network.py index d358229b6..2ce40b4cd 100644 --- a/scripts/plot_power_network.py +++ b/scripts/plot_power_network.py @@ -14,10 +14,11 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config -from plot_summary import preferred_order, rename_techs from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches +from scripts._helpers import configure_logging, set_scenario_config +from scripts.plot_summary import preferred_order, rename_techs + logger = logging.getLogger(__name__) @@ -245,7 +246,7 @@ def plot_map( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_power_network", diff --git a/scripts/plot_power_network_clustered.py b/scripts/plot_power_network_clustered.py index 0c3dc635b..7c510ff3e 100644 --- a/scripts/plot_power_network_clustered.py +++ b/scripts/plot_power_network_clustered.py @@ -9,14 +9,15 @@ import geopandas as gpd import matplotlib.pyplot as plt import pypsa -from _helpers import set_scenario_config from matplotlib.lines import Line2D -from plot_power_network import load_projection from pypsa.plot import add_legend_lines +from scripts._helpers import set_scenario_config +from scripts.plot_power_network import load_projection + if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_power_network_clustered", diff --git a/scripts/plot_power_network_perfect.py b/scripts/plot_power_network_perfect.py index 7d2252be8..92e5d785d 100644 --- a/scripts/plot_power_network_perfect.py +++ b/scripts/plot_power_network_perfect.py @@ -13,11 +13,16 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config -from plot_power_network import assign_location, load_projection, rename_techs_tyndp -from plot_summary import preferred_order from pypsa.plot import add_legend_circles, add_legend_lines +from scripts._helpers import configure_logging, set_scenario_config +from scripts.plot_power_network import ( + assign_location, + load_projection, + rename_techs_tyndp, +) +from scripts.plot_summary import preferred_order + logger = logging.getLogger(__name__) @@ -172,7 +177,7 @@ def plot_map_perfect( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_power_network_perfect", diff --git a/scripts/plot_statistics.py b/scripts/plot_statistics.py index cc98a5364..c06f19983 100644 --- a/scripts/plot_statistics.py +++ b/scripts/plot_statistics.py @@ -7,14 +7,15 @@ import matplotlib.pyplot as plt import pypsa import seaborn as sns -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config sns.set_theme("paper", style="whitegrid") if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_elec_statistics", diff --git a/scripts/plot_statistics_comparison.py b/scripts/plot_statistics_comparison.py index 0d9808cfc..3f8612823 100644 --- a/scripts/plot_statistics_comparison.py +++ b/scripts/plot_statistics_comparison.py @@ -10,8 +10,9 @@ import matplotlib.pyplot as plt import pandas as pd import seaborn as sns -from _helpers import configure_logging -from plot_summary import rename_techs + +from scripts._helpers import configure_logging +from scripts.plot_summary import rename_techs logger = logging.getLogger(__name__) sns.set_theme("paper", style="whitegrid") @@ -88,7 +89,7 @@ def read_csv(input, output): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_statistics_comparison", diff --git a/scripts/plot_statistics_single.py b/scripts/plot_statistics_single.py index 2481ad684..9a1883585 100644 --- a/scripts/plot_statistics_single.py +++ b/scripts/plot_statistics_single.py @@ -9,8 +9,9 @@ import matplotlib.pyplot as plt import pandas as pd import seaborn as sns -from _helpers import configure_logging -from plot_summary import rename_techs + +from scripts._helpers import configure_logging +from scripts.plot_summary import rename_techs logger = logging.getLogger(__name__) sns.set_theme("paper", style="whitegrid") @@ -51,7 +52,7 @@ def read_csv(input): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_statistics_single", diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index d131e9378..b1faaa2f3 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -11,8 +11,9 @@ import matplotlib.gridspec as gridspec import matplotlib.pyplot as plt import pandas as pd -from _helpers import configure_logging, set_scenario_config -from prepare_sector_network import co2_emissions_year + +from scripts._helpers import configure_logging, set_scenario_config +from scripts.prepare_sector_network import co2_emissions_year logger = logging.getLogger(__name__) plt.style.use("ggplot") @@ -569,7 +570,7 @@ def plot_carbon_budget_distribution(input_eurostat, options): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("plot_summary") diff --git a/scripts/plot_validation_cross_border_flows.py b/scripts/plot_validation_cross_border_flows.py index 9107726b4..2c03ee778 100644 --- a/scripts/plot_validation_cross_border_flows.py +++ b/scripts/plot_validation_cross_border_flows.py @@ -9,7 +9,8 @@ import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config sns.set_theme("paper", style="whitegrid") @@ -177,7 +178,7 @@ def cross_border_bar(countries, data): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_electricity_prices", diff --git a/scripts/plot_validation_electricity_prices.py b/scripts/plot_validation_electricity_prices.py index b210086dc..853c0c9db 100644 --- a/scripts/plot_validation_electricity_prices.py +++ b/scripts/plot_validation_electricity_prices.py @@ -8,13 +8,14 @@ import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config sns.set_theme("paper", style="whitegrid") if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_electricity_prices", diff --git a/scripts/plot_validation_electricity_production.py b/scripts/plot_validation_electricity_production.py index 6336b7a74..d8c5552d0 100644 --- a/scripts/plot_validation_electricity_production.py +++ b/scripts/plot_validation_electricity_production.py @@ -8,9 +8,10 @@ import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging, set_scenario_config from pypsa.statistics import get_bus_and_carrier +from scripts._helpers import configure_logging, set_scenario_config + sns.set_theme("paper", style="whitegrid") carrier_groups = { @@ -25,7 +26,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "plot_validation_electricity_production", diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index cd4eebbde..25fadf43b 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -62,14 +62,15 @@ import numpy as np import pandas as pd import pypsa -from _helpers import ( +from pypsa.descriptors import expand_series + +from scripts._helpers import ( configure_logging, get, set_scenario_config, update_config_from_wildcards, ) -from add_electricity import load_costs, set_transmission_costs -from pypsa.descriptors import expand_series +from scripts.add_electricity import load_costs, set_transmission_costs idx = pd.IndexSlice @@ -321,7 +322,7 @@ def set_line_nom_max( # %% if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "prepare_network", diff --git a/scripts/prepare_osm_network_release.py b/scripts/prepare_osm_network_release.py index ac6b25354..8ccfa4e6e 100644 --- a/scripts/prepare_osm_network_release.py +++ b/scripts/prepare_osm_network_release.py @@ -8,7 +8,8 @@ import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -96,7 +97,7 @@ def export_clean_csv(df, columns, output_file): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("prepare_osm_network_release") diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 37e0b0643..191108e81 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -12,14 +12,15 @@ import numpy as np import pandas as pd import pypsa -from _helpers import ( +from add_existing_baseyear import add_build_year_to_new_assets +from pypsa.descriptors import expand_series +from six import iterkeys + +from scripts._helpers import ( configure_logging, set_scenario_config, update_config_from_wildcards, ) -from add_existing_baseyear import add_build_year_to_new_assets -from pypsa.descriptors import expand_series -from six import iterkeys logger = logging.getLogger(__name__) @@ -522,7 +523,7 @@ def update_heat_pump_efficiency(n: pypsa.Network, years: List[int]): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "prepare_perfect_foresight", diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 8b59a3d68..728426697 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,28 +17,33 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import ( +from networkx.algorithms import complement +from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation +from pypsa.geo import haversine_pts +from scipy.stats import beta + +from scripts._helpers import ( configure_logging, get, set_scenario_config, update_config_from_wildcards, ) -from add_electricity import calculate_annuity, sanitize_carriers, sanitize_locations -from build_energy_totals import ( +from scripts.add_electricity import ( + calculate_annuity, + sanitize_carriers, + sanitize_locations, +) +from scripts.build_energy_totals import ( build_co2_totals, build_eea_co2, build_eurostat, build_eurostat_co2, ) -from build_transport_demand import transport_degree_factor -from definitions.heat_sector import HeatSector -from definitions.heat_system import HeatSystem -from definitions.heat_system_type import HeatSystemType -from networkx.algorithms import complement -from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation -from prepare_network import maybe_adjust_costs_and_potentials -from pypsa.geo import haversine_pts -from scipy.stats import beta +from scripts.build_transport_demand import transport_degree_factor +from scripts.definitions.heat_sector import HeatSector +from scripts.definitions.heat_system import HeatSystem +from scripts.definitions.heat_system_type import HeatSystemType +from scripts.prepare_network import maybe_adjust_costs_and_potentials spatial = SimpleNamespace() logger = logging.getLogger(__name__) @@ -2712,7 +2717,6 @@ def add_methanol(n, costs): def add_methanol(n, costs): - methanol_options = options["methanol"] if not any(methanol_options.values()): return @@ -4810,7 +4814,7 @@ def add_enhanced_geothermal(n, egs_potentials, egs_overlap, costs): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "prepare_sector_network", diff --git a/scripts/pypsa-de/additional_functionality.py b/scripts/pypsa-de/additional_functionality.py index 2d167bccb..38d8e017d 100644 --- a/scripts/pypsa-de/additional_functionality.py +++ b/scripts/pypsa-de/additional_functionality.py @@ -3,14 +3,14 @@ import logging import pandas as pd -from prepare_sector_network import determine_emission_sectors from xarray import DataArray +from scripts.prepare_sector_network import determine_emission_sectors + logger = logging.getLogger(__name__) def add_capacity_limits(n, investment_year, limits_capacity, sense="maximum"): - for c in n.iterate_components(limits_capacity): logger.info(f"Adding {sense} constraints for {c.list_name}") @@ -18,7 +18,6 @@ def add_capacity_limits(n, investment_year, limits_capacity, sense="maximum"): units = "MWh or tCO2" if c.name == "Store" else "MW" for carrier in limits_capacity[c.name]: - for ct in limits_capacity[c.name][carrier]: if investment_year not in limits_capacity[c.name][carrier][ct].keys(): continue @@ -161,7 +160,6 @@ def add_power_limits(n, investment_year, limits_power_max): def h2_import_limits(n, investment_year, limits_volume_max): - for ct in limits_volume_max["h2_import"]: limit = limits_volume_max["h2_import"][ct][investment_year] * 1e6 @@ -212,7 +210,6 @@ def h2_import_limits(n, investment_year, limits_volume_max): def h2_production_limits(n, investment_year, limits_volume_min, limits_volume_max): - for ct in limits_volume_max["electrolysis"]: if ct not in limits_volume_min["electrolysis"]: logger.warning( @@ -271,7 +268,6 @@ def h2_production_limits(n, investment_year, limits_volume_min, limits_volume_ma def electricity_import_limits(n, investment_year, limits_volume_max): - for ct in limits_volume_max["electricity_import"]: limit = limits_volume_max["electricity_import"][ct][investment_year] * 1e6 @@ -369,7 +365,6 @@ def add_co2limit_country(n, limit_countries, snakemake, debug=False): lhs = [] for port in [col[3:] for col in n.links if col.startswith("bus")]: - links = n.links.index[ (n.links.index.str[:2] == ct) & (n.links[f"bus{port}"] == "co2 atmosphere") @@ -589,9 +584,12 @@ def force_boiler_profiles_existing_per_boiler(n): # will be per unit n.model.add_variables(coords=[decentral_boilers], name="Link-fixed_profile_scaling") - lhs = (1, n.model["Link-p"].loc[:, decentral_boilers]), ( - -boiler_profiles, - n.model["Link-fixed_profile_scaling"], + lhs = ( + (1, n.model["Link-p"].loc[:, decentral_boilers]), + ( + -boiler_profiles, + n.model["Link-fixed_profile_scaling"], + ), ) n.model.add_constraints(lhs, "=", 0, "Link-fixed_profile_scaling") @@ -601,7 +599,6 @@ def force_boiler_profiles_existing_per_boiler(n): def add_h2_derivate_limit(n, investment_year, limits_volume_max): - for ct in limits_volume_max["h2_derivate_import"]: limit = limits_volume_max["h2_derivate_import"][ct][investment_year] * 1e6 @@ -652,7 +649,6 @@ def add_h2_derivate_limit(n, investment_year, limits_volume_max): def adapt_nuclear_output(n): - logger.info( f"limiting german electricity generation from nuclear to 2020 value of 61 TWh" ) @@ -691,7 +687,6 @@ def adapt_nuclear_output(n): def additional_functionality(n, snapshots, snakemake): - logger.info("Adding Ariadne-specific functionality") investment_year = int(snakemake.wildcards.planning_horizons[-4:]) diff --git a/scripts/pypsa-de/build_egon_data.py b/scripts/pypsa-de/build_egon_data.py index 208c2e212..7ac7ff4ff 100644 --- a/scripts/pypsa-de/build_egon_data.py +++ b/scripts/pypsa-de/build_egon_data.py @@ -16,27 +16,19 @@ - resources/heating_technologies_nuts3.geojson: Path to the GeoJSON file where the processed heating technologies data will be saved. """ -import logging - -logger = logging.getLogger(__name__) import json +import logging import re import geopandas as gpd import pandas as pd -from _helpers import configure_logging - -if __name__ == "__main__": - if "snakemake" not in globals(): - import os - import sys - os.chdir(os.path.dirname(os.path.abspath(__file__))) +from scripts._helpers import configure_logging, mock_snakemake - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake +logger = logging.getLogger(__name__) +if __name__ == "__main__": + if "snakemake" not in globals(): snakemake = mock_snakemake( "build_egon_data", run="KN2045_Bal_v4", diff --git a/scripts/pypsa-de/build_existing_chp_de.py b/scripts/pypsa-de/build_existing_chp_de.py index 90bc1d965..856f542fa 100644 --- a/scripts/pypsa-de/build_existing_chp_de.py +++ b/scripts/pypsa-de/build_existing_chp_de.py @@ -18,9 +18,10 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging from powerplantmatching.export import map_country_bus +from scripts._helpers import configure_logging, mock_snakemake + def clean_data(combustion, biomass, geodata): """ @@ -213,10 +214,6 @@ def BP(cap, year): if __name__ == "__main__": if "snakemake" not in globals(): - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "build_existing_chp_de", clusters=27, diff --git a/scripts/pypsa-de/build_mobility_demand.py b/scripts/pypsa-de/build_mobility_demand.py index 729ed0624..fc9c43283 100644 --- a/scripts/pypsa-de/build_mobility_demand.py +++ b/scripts/pypsa-de/build_mobility_demand.py @@ -3,8 +3,9 @@ import pandas as pd +from scripts._helpers import configure_logging, mock_snakemake + logger = logging.getLogger(__name__) -from _helpers import configure_logging def get_transport_data(db): @@ -45,13 +46,6 @@ def get_transport_data(db): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "build_mobility_demand", simpl="", diff --git a/scripts/pypsa-de/build_scenarios.py b/scripts/pypsa-de/build_scenarios.py index c4e41b215..a2e711e3d 100644 --- a/scripts/pypsa-de/build_scenarios.py +++ b/scripts/pypsa-de/build_scenarios.py @@ -5,17 +5,16 @@ # This script reads in data from the IIASA database to create the scenario.yaml file import logging - -from _helpers import configure_logging - -logger = logging.getLogger(__name__) - import os from pathlib import Path import pandas as pd import ruamel.yaml +from scripts._helpers import configure_logging, mock_snakemake + +logger = logging.getLogger(__name__) + def get_transport_growth(df, planning_horizons): try: @@ -225,13 +224,6 @@ def write_to_scenario_yaml(input, output, scenarios, df): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake("build_scenarios") configure_logging(snakemake) diff --git a/scripts/pypsa-de/build_wasserstoff_kernnetz.py b/scripts/pypsa-de/build_wasserstoff_kernnetz.py index c824195ea..1b0999cc8 100644 --- a/scripts/pypsa-de/build_wasserstoff_kernnetz.py +++ b/scripts/pypsa-de/build_wasserstoff_kernnetz.py @@ -23,10 +23,8 @@ from shapely.geometry import LineString, Point, Polygon from shapely.ops import nearest_points -sys.path.insert(0, os.path.abspath("../")) - -from _helpers import configure_logging -from build_gas_network import diameter_to_capacity +from scripts._helpers import configure_logging, mock_snakemake +from scripts.build_gas_network import diameter_to_capacity MANUAL_ADDRESSES = { "Oude Statenzijl": (7.205108658430258, 53.20183834422634), @@ -554,7 +552,7 @@ def filter_kernnetz( path = "../submodules/pypsa-eur/scripts" sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("build_wasserstoff_kernnetz") diff --git a/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py b/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py index 16eacadcd..6b3842668 100644 --- a/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py +++ b/scripts/pypsa-de/cluster_wasserstoff_kernnetz.py @@ -21,11 +21,8 @@ from shapely.geometry import LineString, Point from shapely.ops import transform -paths = ["workflow/submodules/pypsa-eur/scripts", "../submodules/pypsa-eur/scripts"] -for path in paths: - sys.path.insert(0, os.path.abspath(path)) -from _helpers import configure_logging -from cluster_gas_network import load_bus_regions, reindex_pipes +from scripts._helpers import configure_logging, mock_snakemake +from scripts.cluster_gas_network import load_bus_regions, reindex_pipes # Define a function for projecting points to meters project_to_meters = pyproj.Transformer.from_proj( @@ -189,13 +186,6 @@ def aggregate_parallel_pipes(df, aggregate_build_years="mean"): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "cluster_wasserstoff_kernnetz", simpl="", @@ -261,7 +251,6 @@ def aggregate_parallel_pipes(df, aggregate_build_years="mean"): wasserstoff_kernnetz["p_nom_diameter"] = 0 if kernnetz_cf["aggregate_parallel_pipes"]: - reindex_pipes(wasserstoff_kernnetz, prefix="H2 pipeline") wasserstoff_kernnetz = aggregate_parallel_pipes( wasserstoff_kernnetz, kernnetz_cf["aggregate_build_years"] diff --git a/scripts/pypsa-de/export_ariadne_variables.py b/scripts/pypsa-de/export_ariadne_variables.py index b79576ced..dcb3724ff 100644 --- a/scripts/pypsa-de/export_ariadne_variables.py +++ b/scripts/pypsa-de/export_ariadne_variables.py @@ -12,19 +12,10 @@ import pypsa from numpy import isclose -logger = logging.getLogger(__name__) - -paths = [ - "workflow/submodules/pypsa-eur/scripts", - "../submodules/pypsa-eur/scripts", - "../submodules/pypsa-eur/", -] -for path in paths: - sys.path.insert(0, os.path.abspath(path)) - -from _helpers import configure_logging, mute_print -from prepare_sector_network import prepare_costs +from scripts._helpers import configure_logging, mock_snakemake +from scripts.prepare_sector_network import prepare_costs +logger = logging.getLogger(__name__) # Defining global variables TWh2PJ = 3.6 @@ -443,7 +434,6 @@ def _f(**kwargs): def _get_capacities(n, region, cap_func, cap_string="Capacity|"): - kwargs = { "groupby": n.statistics.groupers.get_bus_and_carrier, "at_port": True, @@ -1800,7 +1790,6 @@ def get_secondary_energy(n, region, _industry_demand): def get_final_energy( n, region, _industry_demand, _energy_totals, _sector_ratios, _industry_production ): - var = pd.Series() kwargs = { @@ -3161,7 +3150,6 @@ def get_weighted_costs_links(carriers, n, region): def get_weighted_costs(costs, flows): - cleaned_costs = [] cleaned_flows = [] @@ -3758,7 +3746,6 @@ def get_prices(n, region): def get_discretized_value(value, disc_int, build_threshold=0.3): - if value == 0.0: return value @@ -4166,7 +4153,6 @@ def get_policy(n, investment_year): def get_economy(n, region): - var = pd.Series() s = n.statistics @@ -4441,7 +4427,6 @@ def get_export_import_links(n, region, carriers): def get_production(region, year): - var = pd.Series() # read in the industrial production data years = [ @@ -4695,7 +4680,6 @@ def get_operational_and_capital_costs(year): def get_grid_capacity(n, region, year): - var = pd.Series() ### Total Capacity ## Transmission Grid @@ -4908,7 +4892,6 @@ def get_grid_capacity(n, region, year): def hack_DC_projects(n, p_nom_start, p_nom_planned, model_year, snakemake, costs): - logger.info(f"Hacking DC projects for year {model_year}") logger.info(f"Assuming all indices of DC projects start with 'DC' or 'TYNDP'") @@ -5065,7 +5048,6 @@ def get_ariadne_var( region, year, ): - var = pd.concat( [ get_capacities(n, region), @@ -5113,7 +5095,6 @@ def get_data( version="0.10", scenario="test", ): - var = get_ariadne_var( n, industry_demand, @@ -5186,13 +5167,6 @@ def get_data( if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "export_ariadne_variables", simpl="", diff --git a/scripts/pypsa-de/modify_cost_data.py b/scripts/pypsa-de/modify_cost_data.py index 2ce1654b7..22dbefe82 100644 --- a/scripts/pypsa-de/modify_cost_data.py +++ b/scripts/pypsa-de/modify_cost_data.py @@ -3,10 +3,12 @@ import logging import os import re +import sys import numpy as np import pandas as pd -from _helpers import configure_logging + +from scripts._helpers import configure_logging, mock_snakemake logger = logging.getLogger(__name__) @@ -41,12 +43,6 @@ def carbon_component_fossils(costs, co2_price): if __name__ == "__main__": if "snakemake" not in globals(): - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "modify_cost_data", planning_horizons="2020", diff --git a/scripts/pypsa-de/modify_district_heat_share.py b/scripts/pypsa-de/modify_district_heat_share.py index 565bbf1a6..942c7dfe9 100644 --- a/scripts/pypsa-de/modify_district_heat_share.py +++ b/scripts/pypsa-de/modify_district_heat_share.py @@ -22,11 +22,14 @@ import logging -logger = logging.getLogger(__name__) import geopandas as gpd import pandas as pd from shapely.geometry import Point +from scripts._helpers import mock_snakemake + +logger = logging.getLogger(__name__) + def cluster_egon(heat_techs, regions_onshore): """ @@ -96,15 +99,6 @@ def update_district_heat_share(heat_techs_clustered, dh_shares): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "modify_district_heat_share", simpl="", diff --git a/scripts/pypsa-de/modify_existing_heating.py b/scripts/pypsa-de/modify_existing_heating.py index 8ff24833a..d3cb06d09 100644 --- a/scripts/pypsa-de/modify_existing_heating.py +++ b/scripts/pypsa-de/modify_existing_heating.py @@ -2,22 +2,14 @@ import logging import pandas as pd -from _helpers import configure_logging + +from scripts._helpers import configure_logging, mock_snakemake logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "modify_existing_heating", run="KN2045_Bal_v4", diff --git a/scripts/pypsa-de/modify_industry_demand.py b/scripts/pypsa-de/modify_industry_demand.py index 33500a715..9fb812cbb 100644 --- a/scripts/pypsa-de/modify_industry_demand.py +++ b/scripts/pypsa-de/modify_industry_demand.py @@ -14,21 +14,16 @@ - Production|Non-Ferrous Metals - Production|Pulp and Paper """ + import logging logger = logging.getLogger(__name__) import pandas as pd -from _helpers import configure_logging + +from scripts._helpers import configure_logging, mock_snakemake if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "modify_industry_demand", simpl="", diff --git a/scripts/pypsa-de/modify_prenetwork.py b/scripts/pypsa-de/modify_prenetwork.py index 9338e0a10..4597f94df 100644 --- a/scripts/pypsa-de/modify_prenetwork.py +++ b/scripts/pypsa-de/modify_prenetwork.py @@ -9,16 +9,11 @@ import pypsa from shapely.geometry import Point -logger = logging.getLogger(__name__) - -sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..") +from scripts._helpers import configure_logging, mock_snakemake +from scripts.add_electricity import load_costs +from scripts.prepare_sector_network import lossy_bidirectional_links, prepare_costs -paths = ["workflow/submodules/pypsa-eur/scripts", "../submodules/pypsa-eur/scripts"] -for path in paths: - sys.path.insert(0, os.path.abspath(path)) -from _helpers import configure_logging -from add_electricity import load_costs -from prepare_sector_network import lossy_bidirectional_links, prepare_costs +logger = logging.getLogger(__name__) def first_technology_occurrence(n): @@ -1262,13 +1257,6 @@ def scale_capacity(n, scaling): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "modify_prenetwork", simpl="", diff --git a/scripts/pypsa-de/plot_ariadne_report.py b/scripts/pypsa-de/plot_ariadne_report.py index 15f6d6ab0..977a9ed1e 100644 --- a/scripts/pypsa-de/plot_ariadne_report.py +++ b/scripts/pypsa-de/plot_ariadne_report.py @@ -18,14 +18,12 @@ from matplotlib.lines import Line2D from pypsa.plot import add_legend_lines -path = "../submodules/pypsa-eur/scripts" -sys.path.insert(1, os.path.abspath(path)) -from _helpers import configure_logging, set_scenario_config -from export_ariadne_variables import get_discretized_value, process_postnetworks -from plot_power_network import load_projection -from plot_summary import preferred_order, rename_techs -from prepare_sector_network import prepare_costs -from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches +from scripts._helpers import configure_logging, mock_snakemake, set_scenario_config +from scripts.export_ariadne_variables import get_discretized_value, process_postnetworks +from scripts.plot_power_network import load_projection +from scripts.plot_summary import preferred_order, rename_techs +from scripts.prepare_sector_network import prepare_costs +from scripts.pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches logger = logging.getLogger(__name__) @@ -215,7 +213,6 @@ def plot_nodal_balance( ylabel="total electricity balance [GW]", title="Electricity balance", ): - carriers = carriers loads = loads start_date = start_date @@ -259,9 +256,10 @@ def plot_nodal_balance( df_pos = df_pos[df_pos.sum().sort_values(ascending=False).index] df_neg = df_neg[df_neg.sum().sort_values().index] # get colors - c_neg, c_pos = [tech_colors[col] for col in df_neg.columns], [ - tech_colors[col] for col in df_pos.columns - ] + c_neg, c_pos = ( + [tech_colors[col] for col in df_neg.columns], + [tech_colors[col] for col in df_pos.columns], + ) fig, ax = plt.subplots(figsize=(14, 8)) @@ -470,7 +468,6 @@ def plot_storage( end_date="2019-12-31 00", regions=["DE"], ): - # State of charge [per unit of max] (all stores and storage units) # Ratio of total generation of max state of charge @@ -599,7 +596,6 @@ def plot_price_duration_curve( y_lim_values=[-50, 300], language="english", ): - fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(8, 6)) for i, n in enumerate(networks.values()): @@ -660,7 +656,6 @@ def plot_price_duration_hist( regions=["DE"], x_lim_values=[-50, 300], ): - fig, axes = plt.subplots(ncols=1, nrows=len(years), figsize=(8, 3 * len(years))) axes = axes.flatten() @@ -950,7 +945,6 @@ def plot_h2_map(n, regions, savepath, only_de=False): def plot_h2_map_de(n, regions, tech_colors, savepath, specify_buses=None): - assign_location(n) h2_storage = n.stores[n.stores.carrier.isin(["H2", "H2 Store"])] @@ -1286,7 +1280,6 @@ def plot_elec_map_de( savepath, expansion_case="total-expansion", ): - m = network.copy() m.mremove("Bus", m.buses[m.buses.x == 0].index) m.buses.drop(m.buses.index[m.buses.carrier != "AC"], inplace=True) @@ -1462,13 +1455,6 @@ def plot_elec_map_de( if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "plot_ariadne_report", simpl="", diff --git a/scripts/pypsa-de/plot_ariadne_scenario_comparison.py b/scripts/pypsa-de/plot_ariadne_scenario_comparison.py index ba78e3217..3c4796c2a 100644 --- a/scripts/pypsa-de/plot_ariadne_scenario_comparison.py +++ b/scripts/pypsa-de/plot_ariadne_scenario_comparison.py @@ -5,6 +5,8 @@ import matplotlib.pyplot as plt import pandas as pd +from scripts._helpers import mock_snakemake + def scenario_plot(df, var): unit = df._get_label_or_level_values("Unit")[0] @@ -20,12 +22,6 @@ def scenario_plot(df, var): if __name__ == "__main__": if "snakemake" not in globals(): - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "ariadne_all", # simpl="", diff --git a/scripts/pypsa-de/plot_ariadne_variables.py b/scripts/pypsa-de/plot_ariadne_variables.py index e9fc8dab3..07cfe6bc7 100644 --- a/scripts/pypsa-de/plot_ariadne_variables.py +++ b/scripts/pypsa-de/plot_ariadne_variables.py @@ -5,6 +5,8 @@ import numpy as np import pandas as pd +from scripts._helpers import mock_snakemake + def plot_Kernnetz(df, savepath=None, currency_year=2020): key = "Investment|Energy Supply|Hydrogen|Transmission and Distribution|" @@ -102,7 +104,6 @@ def plot_Kernnetz(df, savepath=None, currency_year=2020): def plot_NEP_Trassen(df, savepath=None, gleichschaltung=True): - NEP_Trassen = { "NEP-DC": { # Zu-/Umbeseilung + Ersatz-/Parallelneubau + Neubau @@ -203,7 +204,6 @@ def plot_NEP_Trassen(df, savepath=None, gleichschaltung=True): def plot_NEP(df, savepath=None, gleichschaltung=True, currency_year=2020): - key = "Investment|Energy Supply|Electricity|Transmission|" data = { @@ -427,7 +427,6 @@ def ariadne_subplot( def side_by_side_plot(df, dfhybrid, title, savepath, rshift=1.25, **kwargs): - idx = df.index.union(dfhybrid.index, sort=False) df = df.reindex(idx) @@ -441,7 +440,9 @@ def side_by_side_plot(df, dfhybrid, title, savepath, rshift=1.25, **kwargs): labels2 = ax2.get_legend_handles_labels()[1] assert labels == labels2 - fig.legend(reversed(handles), reversed(labels), bbox_to_anchor=(rshift, 0.9)) + fig.legend( + list(reversed(handles)), list(reversed(labels)), bbox_to_anchor=(rshift, 0.9) + ) fig.suptitle(title) title = title.replace(" ", "_") fig.savefig(savepath, bbox_inches="tight") @@ -459,7 +460,6 @@ def within_plot( unit="EUR_2020/GJ", **kwargs, ): - df = df.T.copy() if select_regex: df = df.filter( @@ -487,7 +487,6 @@ def within_plot( axes = axes.flatten() for i, var in enumerate(df.columns.get_level_values("Variable")): - axes[i].plot(df.xs(var, axis=1, level=0), label="PyPSA-Eur") if var in df2.index.get_level_values("Variable"): axes[i].plot(df2.T.xs(var, axis=1, level=0), label="REMIND-EU") @@ -668,13 +667,6 @@ def elec_val_plot(df, savepath): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake( "plot_ariadne_variables", simpl="", diff --git a/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py b/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py index f5fe6244e..3a353cb39 100644 --- a/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py +++ b/scripts/pypsa-de/plot_hydrogen_network_incl_kernnetz.py @@ -8,21 +8,17 @@ """ import logging -import os -import sys import geopandas as gpd import matplotlib.pyplot as plt import numpy as np import pandas as pd import pypsa - -path = "../submodules/pypsa-eur/scripts" -sys.path.insert(0, os.path.abspath(path)) -from _helpers import configure_logging, set_scenario_config -from plot_power_network import assign_location, load_projection from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches +from scripts._helpers import configure_logging, mock_snakemake, set_scenario_config +from scripts.plot_power_network import assign_location, load_projection + logger = logging.getLogger(__name__) @@ -331,11 +327,6 @@ def plot_h2_map(n, regions): if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - from _helpers import mock_snakemake - snakemake = mock_snakemake( "plot_hydrogen_network_incl_kernnetz", simpl="", diff --git a/scripts/pypsa-de/retrieve_ariadne_database.py b/scripts/pypsa-de/retrieve_ariadne_database.py index 50e6600ea..09f9d1ba1 100644 --- a/scripts/pypsa-de/retrieve_ariadne_database.py +++ b/scripts/pypsa-de/retrieve_ariadne_database.py @@ -2,19 +2,13 @@ import logging import pyam -from _helpers import configure_logging + +from scripts._helpers import configure_logging, mock_snakemake logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - import os - import sys - - path = "../submodules/pypsa-eur/scripts" - sys.path.insert(0, os.path.abspath(path)) - from _helpers import mock_snakemake - snakemake = mock_snakemake("retrieve_ariadne_database") configure_logging(snakemake) diff --git a/scripts/retrieve_cost_data.py b/scripts/retrieve_cost_data.py index e236fbfdb..56792377b 100644 --- a/scripts/retrieve_cost_data.py +++ b/scripts/retrieve_cost_data.py @@ -9,13 +9,13 @@ import logging from pathlib import Path -from _helpers import configure_logging, progress_retrieve, set_scenario_config +from scripts._helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_cost_data", year=2030) rootpath = ".." diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 95856f385..98c17945e 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -27,7 +27,7 @@ import tarfile from pathlib import Path -from _helpers import ( +from scripts._helpers import ( configure_logging, progress_retrieve, set_scenario_config, @@ -39,7 +39,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_databundle") rootpath = ".." diff --git a/scripts/retrieve_electricity_demand.py b/scripts/retrieve_electricity_demand.py index d4b8d758f..384f755e8 100644 --- a/scripts/retrieve_electricity_demand.py +++ b/scripts/retrieve_electricity_demand.py @@ -9,13 +9,14 @@ import logging import pandas as pd -from _helpers import configure_logging, set_scenario_config + +from scripts._helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_electricity_demand") rootpath = ".." diff --git a/scripts/retrieve_eurostat_data.py b/scripts/retrieve_eurostat_data.py index 9ee32e6a5..6eef60802 100644 --- a/scripts/retrieve_eurostat_data.py +++ b/scripts/retrieve_eurostat_data.py @@ -6,18 +6,17 @@ Retrieve and extract eurostat energy balances data. """ - import logging import zipfile from pathlib import Path -from _helpers import configure_logging, progress_retrieve, set_scenario_config +from scripts._helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_eurostat_data") rootpath = ".." diff --git a/scripts/retrieve_eurostat_household_data.py b/scripts/retrieve_eurostat_household_data.py index 0ad9e34af..70bcfda19 100644 --- a/scripts/retrieve_eurostat_household_data.py +++ b/scripts/retrieve_eurostat_household_data.py @@ -6,19 +6,18 @@ Retrieve and extract eurostat household energy balances data. """ - import gzip import logging import shutil from pathlib import Path -from _helpers import configure_logging, progress_retrieve, set_scenario_config +from scripts._helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_eurostat_data") rootpath = ".." diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index 972e08c5b..21ae80c8f 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -11,7 +11,7 @@ import zipfile from pathlib import Path -from _helpers import ( +from scripts._helpers import ( configure_logging, progress_retrieve, set_scenario_config, @@ -23,7 +23,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_gas_network_data") rootpath = ".." diff --git a/scripts/retrieve_jrc_idees.py b/scripts/retrieve_jrc_idees.py index 1e56e6363..30333f359 100644 --- a/scripts/retrieve_jrc_idees.py +++ b/scripts/retrieve_jrc_idees.py @@ -11,7 +11,7 @@ import zipfile from pathlib import Path -from _helpers import configure_logging, progress_retrieve, set_scenario_config +from scripts._helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) @@ -20,7 +20,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_jrc_idees") rootpath = ".." diff --git a/scripts/retrieve_monthly_fuel_prices.py b/scripts/retrieve_monthly_fuel_prices.py index 74861c42f..89b88b67d 100644 --- a/scripts/retrieve_monthly_fuel_prices.py +++ b/scripts/retrieve_monthly_fuel_prices.py @@ -9,13 +9,13 @@ import logging from pathlib import Path -from _helpers import configure_logging, progress_retrieve, set_scenario_config +from scripts._helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_monthly_fuel_prices") rootpath = ".." diff --git a/scripts/retrieve_osm_data.py b/scripts/retrieve_osm_data.py index 745533cff..47e78cf7e 100644 --- a/scripts/retrieve_osm_data.py +++ b/scripts/retrieve_osm_data.py @@ -17,7 +17,8 @@ import time import requests -from _helpers import ( # set_scenario_config,; update_config_from_wildcards,; update_config_from_wildcards, + +from scripts._helpers import ( # set_scenario_config,; update_config_from_wildcards,; update_config_from_wildcards, configure_logging, set_scenario_config, ) @@ -139,7 +140,7 @@ def retrieve_osm_data( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("retrieve_osm_data", country="BE") configure_logging(snakemake) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 1aa455428..ed4f364f4 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -75,12 +75,13 @@ import pandas as pd import pypsa import scipy as sp -from _helpers import configure_logging, set_scenario_config from base_network import append_bus_shapes from cluster_network import cluster_regions from pypsa.clustering.spatial import busmap_by_stubs, get_clustering_from_busmap from scipy.sparse.csgraph import connected_components, dijkstra +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) @@ -414,7 +415,7 @@ def remove_converters(n: pypsa.Network) -> pypsa.Network: if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake("simplify_network") configure_logging(snakemake) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index f2755043e..2ed039d6e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -26,6 +26,7 @@ the workflow for all scenarios in the configuration file (``scenario:``) based on the rule :mod:`solve_network`. """ + import importlib import logging import os @@ -39,14 +40,15 @@ import xarray as xr import yaml from _benchmark import memory_logger -from _helpers import ( +from pypsa.descriptors import get_activity_mask +from pypsa.descriptors import get_switchable_as_dense as get_as_dense + +from scripts._helpers import ( configure_logging, set_scenario_config, update_config_from_wildcards, ) -from prepare_sector_network import get -from pypsa.descriptors import get_activity_mask -from pypsa.descriptors import get_switchable_as_dense as get_as_dense +from scripts.prepare_sector_network import get logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -1056,7 +1058,7 @@ def solve_network(n, config, params, solving, **kwargs): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "solve_sector_network_perfect", diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 28e0c1b79..88b6b4553 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -7,24 +7,24 @@ previous capacity expansion in rule :mod:`solve_network`. """ - import logging import numpy as np import pypsa -from _helpers import ( +from solve_network import prepare_network, solve_network + +from scripts._helpers import ( configure_logging, set_scenario_config, update_config_from_wildcards, ) -from solve_network import prepare_network, solve_network logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "solve_operations_network", diff --git a/scripts/time_aggregation.py b/scripts/time_aggregation.py index 51ab79545..e8cb06d5a 100644 --- a/scripts/time_aggregation.py +++ b/scripts/time_aggregation.py @@ -40,7 +40,6 @@ data is done in ``prepare_sector_network.py``. """ - import logging import numpy as np @@ -48,7 +47,8 @@ import pypsa import tsam.timeseriesaggregation as tsam import xarray as xr -from _helpers import ( + +from scripts._helpers import ( configure_logging, set_scenario_config, update_config_from_wildcards, @@ -58,7 +58,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "time_aggregation", diff --git a/scripts/write_statistics.py b/scripts/write_statistics.py index cf57e311d..d1c4243aa 100644 --- a/scripts/write_statistics.py +++ b/scripts/write_statistics.py @@ -10,9 +10,10 @@ import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging from pypsa.statistics import get_carrier, get_country_and_carrier +from scripts._helpers import configure_logging + logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ def call_with_handle(func, **kwargs): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "write_statistics",