From 345a3e278aec2418edac3cd9e05d1366c842e0eb Mon Sep 17 00:00:00 2001 From: pesap Date: Wed, 18 Sep 2024 23:49:45 -0600 Subject: [PATCH 01/11] fix: Propagating value curves to reeds parser --- src/r2x/defaults/plexos_output.json | 21 +++++++------- src/r2x/exporter/handler.py | 5 ++++ src/r2x/exporter/plexos.py | 17 +++++------ src/r2x/exporter/utils.py | 11 ++++++-- src/r2x/models/costs.py | 44 ++++++++++++++++------------- src/r2x/parser/reeds.py | 33 ++++++++++++++++++++++ src/r2x/units.py | 4 +++ 7 files changed, 96 insertions(+), 39 deletions(-) diff --git a/src/r2x/defaults/plexos_output.json b/src/r2x/defaults/plexos_output.json index 8c6808c7..8592c7bc 100644 --- a/src/r2x/defaults/plexos_output.json +++ b/src/r2x/defaults/plexos_output.json @@ -76,6 +76,7 @@ "min_up_time": "Min Up Time" }, "plexos_property_map": { + "active_power": "Max Capacity", "available": "Units", "base_power": "Max Capacity", "base_voltage": "Voltage", @@ -135,24 +136,24 @@ }, "reserve_types": { "1": { - "direction": "up", - "type": "spinning" + "direction": "UP", + "type": "SPINNING" }, "2": { - "direction": "down", - "type": "spinning" + "direction": "DOWN", + "type": "SPINNING" }, "3": { - "direction": "up", - "type": "regulation" + "direction": "UP", + "type": "REGULATION" }, "4": { - "direction": "down", - "type": "regulation" + "direction": "DOWN", + "type": "REGULATION" }, "default": { - "direction": "up", - "type": "spinning" + "direction": "UP", + "type": "SPINNING" } }, "spin_reserve_file_prefix": "Spin_reserve", diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index 56d73d39..85980593 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -174,6 +174,11 @@ def get_valid_records_properties( ] for component in component_list_mapped: component_dict = {"name": component["name"]} # We need the name to match it with the membership. + if ext_dict := component.get("ext"): + if ext_dict.get("heat_rate"): + component["Heat Rate"] = ext_dict["heat_rate"] + if ext_dict.get("fuel_price"): + component["Fuel Price"] = ext_dict["fuel_price"] for property_name, property_value in component.items(): if valid_properties is not None: if property_name in valid_properties: diff --git a/src/r2x/exporter/plexos.py b/src/r2x/exporter/plexos.py index 651dad96..d4b4fe9e 100644 --- a/src/r2x/exporter/plexos.py +++ b/src/r2x/exporter/plexos.py @@ -166,7 +166,7 @@ def insert_component_properties( filter_func: Callable | None = None, scenario: str | None = None, records: list[dict] | None = None, - exclude_fields: list[str] = NESTED_ATTRIBUTES, + exclude_fields: list[str] | None = NESTED_ATTRIBUTES, ) -> None: """Bulk insert properties from selected component type.""" logger.debug("Adding {} table properties...", component_type.__name__) @@ -184,10 +184,10 @@ def insert_component_properties( collection_properties = self._db_mgr.get_valid_properties( collection, parent_class=parent_class, child_class=child_class ) - property_names = [key[0] for key in collection_properties] + # property_names = [key[0] for key in collection_properties] match component_type.__name__: case "GenericBattery": - custom_map = {"base_power": "Max Power", "storage_capacity": "Capacity"} + custom_map = {"active_power": "Max Power", "storage_capacity": "Capacity"} case _: custom_map = {} property_map = self.property_map | custom_map @@ -195,7 +195,7 @@ def insert_component_properties( records, property_map, self.default_units, - valid_properties=property_names, + valid_properties=collection_properties, ) self._db_mgr.add_property_from_records( valid_component_properties, @@ -219,8 +219,9 @@ def add_component_category( } existing_rank = self._db_mgr.get_category_max_id(class_enum) + class_id = self._db_mgr.get_class_id(class_enum) categories = [ - (class_enum.value, rank, category or "") + (class_id, rank, category or "") for rank, category in enumerate(sorted(component_categories), start=existing_rank + 1) ] @@ -594,6 +595,7 @@ def exclude_battery(component): parent_class=ClassEnum.System, collection=CollectionEnum.Generators, filter_func=exclude_battery, + exclude_fields=[], ) # Add generator memberships @@ -869,13 +871,12 @@ def get_valid_component_properties( # collection_properties = self._db_mgr.query( # f"select name, property_id from t_property where collection_id={collection}" # ) - collection_properties = self._db_mgr.get_valid_properties( + valid_collection_properties = self._db_mgr.get_valid_properties( collection, parent_class=parent_class, child_class=child_class ) - valid_properties = [key[0] for key in collection_properties] for property_name, property_value in component_dict_mapped.items(): - if property_name in valid_properties: + if property_name in valid_collection_properties: property_value = get_property_magnitude(property_value, to_unit=unit_map.get(property_name)) valid_component_properties[property_name] = property_value return valid_component_properties diff --git a/src/r2x/exporter/utils.py b/src/r2x/exporter/utils.py index ec44de92..69c2b8f7 100644 --- a/src/r2x/exporter/utils.py +++ b/src/r2x/exporter/utils.py @@ -6,8 +6,15 @@ def get_reserve_type( reserve_type: ReserveType, reserve_direction: ReserveDirection, reserve_types: dict[str, dict[str, str]] ) -> str: - """Return the reserve type from a mapping.""" + """Return the reserve type from a mapping. + + If not found, return default reserve type + """ for key, value in reserve_types.items(): if value["type"] == reserve_type and value["direction"] == reserve_direction: return key - return "default" + return get_reserve_type( + ReserveType[reserve_types["default"]["type"]], + ReserveDirection[reserve_types["default"]["direction"]], + reserve_types, + ) diff --git a/src/r2x/models/costs.py b/src/r2x/models/costs.py index 0c5d6735..2eceb480 100644 --- a/src/r2x/models/costs.py +++ b/src/r2x/models/costs.py @@ -4,7 +4,7 @@ from typing import Annotated from pydantic import Field from infrasys.cost_curves import ProductionVariableCostCurve -from r2x.units import FuelPrice +from r2x.units import Currency, FuelPrice class OperationalCost(Component): @@ -17,34 +17,40 @@ class RenewableGenerationCost(OperationalCost): class HydroGenerationCost(OperationalCost): - fixed: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice(0.0, "usd/MWh") + fixed: Annotated[ + Currency | None, + Field( + description=( + "Fixed cost of keeping the unit online. " + "For some cost represenations this field can be duplicative" + ) + ), + ] = Currency(0, "usd") variable: ProductionVariableCostCurve | None = None class ThermalGenerationCost(OperationalCost): - start_up: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice( - 0.0, "usd/MWh" - ) fixed: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice(0.0, "usd/MWh") - shut_down: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice( - 0.0, "usd/MWh" + shut_down: Annotated[Currency | None, Field(description="Cost to turn the unit off")] = Currency( + 0.0, "usd" ) + start_up: Annotated[Currency | None, Field(description="Cost to start the unit.")] = Currency(0, "usd") variable: ProductionVariableCostCurve | None = None class StorageCost(OperationalCost): - start_up: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice( - 0.0, "usd/MWh" - ) - fixed: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice(0.0, "usd/MWh") - shut_down: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice( - 0.0, "usd/MWh" + charge_variable_cost: ProductionVariableCostCurve | None = None + discharge_variable_cost: ProductionVariableCostCurve | None = None + energy_shortage_cost: Annotated[ + Currency, Field(description="Cost incurred by the model for being short of the energy target") + ] = Currency(0.0, "usd") + energy_surplus_cost: Annotated[Currency, Field(description="Cost of using fuel in $/MWh.")] = Currency( + 0.0, "usd" ) - energy_surplus_cost: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice( - 0.0, "usd/MWh" + fixed: Annotated[Currency, Field(description=" Fixed cost of operating the storage system")] = Currency( + 0.0, "usd" ) - energy_storage_cost: Annotated[FuelPrice, Field(description="Cost of using fuel in $/MWh.")] = FuelPrice( - 0.0, "usd/MWh" + shut_down: Annotated[Currency | None, Field(description="Cost to turn the unit off")] = Currency( + 0.0, "usd" ) - charge_variable_cost: ProductionVariableCostCurve | None = None - discharge_variable_cost: ProductionVariableCostCurve | None = None + start_up: Annotated[Currency | None, Field(description="Cost to start the unit.")] = Currency(0, "usd") diff --git a/src/r2x/parser/reeds.py b/src/r2x/parser/reeds.py index ca92b0b3..062336b9 100644 --- a/src/r2x/parser/reeds.py +++ b/src/r2x/parser/reeds.py @@ -7,6 +7,9 @@ from operator import attrgetter from argparse import ArgumentParser +from infrasys.cost_curves import CostCurve, FuelCurve +from infrasys.function_data import LinearFunctionData +from infrasys.value_curves import AverageRateCurve, LinearCurve import numpy as np import polars as pl import pyarrow as pa @@ -35,6 +38,8 @@ TransmissionInterfaceMap, ) from r2x.models.core import MinMax +from r2x.models.costs import HydroGenerationCost, ThermalGenerationCost +from r2x.models.generators import RenewableGen, ThermalGen from r2x.parser.handler import BaseParser from r2x.units import ActivePower, EmissionRate, Energy, Percentage, Time, ureg from r2x.utils import match_category, read_csv @@ -420,6 +425,32 @@ def _construct_generators(self) -> None: # noqa: C901 for reserve_type in row["services"]: reserve_map.mapping[reserve_type.name].append(row["name"]) + # Add operational cost data + # ReEDS model all the thermal generators assuming an average heat rate + if issubclass(gen_model, RenewableGen): + row["operation_cost"] = None + if issubclass(gen_model, ThermalGen): + if heat_rate := row.get("heat_rate"): + heat_rate_curve = AverageRateCurve( + function_data=LinearFunctionData( + proportional_term=heat_rate.magnitude, + constant_term=0, + ), + initial_input=heat_rate.magnitude, + ) + fuel_curve = FuelCurve( + value_curve=heat_rate_curve, + vom_cost=LinearCurve(row.get("vom_price", None)), + fuel_cost=row.get("fuel_price", None), + ) + row["operation_cost"] = ThermalGenerationCost( + variable=fuel_curve, + ) + if issubclass(gen_model, HydroGen): + row["operation_cost"] = HydroGenerationCost( + variable=CostCurve(value_curve=LinearCurve(row.get("vom_price", None))) + ) + valid_fields = { key: value for key, value in row.items() if key in gen_model.model_fields if value is not None } @@ -433,6 +464,8 @@ def _construct_generators(self) -> None: # noqa: C901 "reeds_tech": row["tech"], "reeds_vintage": row["tech_vintage"], "Commit": commit, + "heat_rate": row.get("heat_rate"), + "fuel_price": row.get("fuel_price"), } ) diff --git a/src/r2x/units.py b/src/r2x/units.py index dc0adbb7..3f23ff6c 100644 --- a/src/r2x/units.py +++ b/src/r2x/units.py @@ -65,6 +65,10 @@ class PowerRate(BaseQuantity): __base_unit__ = "MW/min" +class Currency(BaseQuantity): + __base_unit__ = "usd" + + def get_magnitude(field) -> float | int: """Get reference base power of the component.""" return field.magnitude if isinstance(field, BaseQuantity) else field From 9009241b2a6466e3c2f5e010fca5f05e863b7c28 Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 19 Sep 2024 00:04:18 -0600 Subject: [PATCH 02/11] fix: Using temp infrasys before realease --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f818030e..96a1069f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ build-backend = "hatchling.build" [tool.hatch.version] path = "src/r2x/__version__.py" +[tool.hatch.metadata] +allow-direct-references=true + [project] name = "r2x" dynamic = ["version"] @@ -25,10 +28,10 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ + "infrasys @ git+https://github.com/NREL/infrasys@ps/value_curves", "jsonschema~=4.23", "loguru~=0.7.2", "pandas~=2.2", - "infrasys>=0.1.0", "plexosdb>=0.0.4", "polars~=1.1.0", "pyyaml~=6.0.1", From 98eec540109194904e6ea973b0ebd6f4cf3bcc4c Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 19 Sep 2024 00:09:25 -0600 Subject: [PATCH 03/11] tests: fix testing --- src/r2x/api.py | 2 +- tests/test_reeds_parser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/r2x/api.py b/src/r2x/api.py index 1fb6b1b1..9ed78776 100644 --- a/src/r2x/api.py +++ b/src/r2x/api.py @@ -85,7 +85,7 @@ def _add_operation_cost_data( # noqa: C901 "uuid" in cost_field_value.keys() ), f"Operation cost field {cost_field_key} was assumed to be a component but is not." variable_cost = self.get_component_by_uuid(uuid.UUID(cost_field_value["uuid"])) - sub_dict["variable_cost"] = variable_cost.vom_units.function_data.proportional_term + sub_dict["variable_cost"] = variable_cost.vom_cost.function_data.proportional_term if "fuel_cost" in variable_cost.model_fields: # Note: We multiply the fuel price by 1000 to offset the division # done by Sienna when it parses .csv files diff --git a/tests/test_reeds_parser.py b/tests/test_reeds_parser.py index 3885379c..930fab48 100644 --- a/tests/test_reeds_parser.py +++ b/tests/test_reeds_parser.py @@ -39,7 +39,7 @@ def test_system_creation(reeds_parser_instance): def test_construct_generators(reeds_parser_instance): - reeds_parser_instance.system = System(name="Test") + reeds_parser_instance.system = System(name="Test", auto_add_composed_components=True) reeds_parser_instance._construct_buses() reeds_parser_instance._construct_reserves() reeds_parser_instance._construct_generators() From f88c34c22a79e43f16e139c8013a7f9c13994572 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Thu, 19 Sep 2024 16:17:15 -0700 Subject: [PATCH 04/11] WIP implements infrasys to plexos for FuelCurves --- src/r2x/defaults/plexos_input.json | 2 +- src/r2x/exporter/handler.py | 38 ++++++++++++++++++++++++++++++ src/r2x/exporter/plexos.py | 10 ++++++-- src/r2x/models/costs.py | 6 +++-- src/r2x/parser/plexos.py | 9 ++++--- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/r2x/defaults/plexos_input.json b/src/r2x/defaults/plexos_input.json index 3b3587fc..80b4cdc3 100644 --- a/src/r2x/defaults/plexos_input.json +++ b/src/r2x/defaults/plexos_input.json @@ -1,7 +1,7 @@ { "plexos_device_map": {}, "plexos_fuel_map": {}, - "plexos_property_map": { + "plexos_input_property_map": { "Charge Efficiency": "charge_efficiency", "Commit": "must_run", "Discharge Efficiency": "discharge_efficiency", diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index 85980593..bf910ea0 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -8,15 +8,22 @@ from typing import Any import pint +from pint import Quantity import pandas as pd import numpy as np import infrasys from loguru import logger from infrasys.base_quantity import BaseQuantity +from infrasys.function_data import ( + LinearFunctionData, + QuadraticFunctionData, +) +from infrasys.value_curves import InputOutputCurve, AverageRateCurve from r2x.api import System from r2x.config import Scenario from r2x.parser.handler import file_handler +from r2x.models.costs import ThermalGenerationCost, RenewableGenerationCost, HydroGenerationCost OUTPUT_FNAME = "{self.weather_year}" @@ -160,6 +167,35 @@ def export_data_files(self, time_series_folder: str = "Data") -> None: return + def parse_operation_cost(self, component): + """Parse Infrasys Operation Cost into Plexos Records.""" + op_cost = component.get("operation_cost") + if isinstance(op_cost, ThermalGenerationCost): + fuel_curve = op_cost.variable + if fuel_curve: + hr_curve = fuel_curve.value_curve + if isinstance(hr_curve, AverageRateCurve): + component["Heat Rate"] = Quantity(hr_curve.function_data.proportional_term) + elif isinstance(hr_curve, InputOutputCurve): + fn = hr_curve.function_data + if isinstance(fn, QuadraticFunctionData): + component["Heat Rate Base"] = Quantity(fn.constant_term) + component["Heat Rate Incr"] = Quantity(fn.proportional_term) + component["Heat Rate Incr2"] = Quantity(fn.quadratic_term) + elif isinstance(fn, LinearFunctionData): + component["Heat Rate Base"] = Quantity(fn.constant_term) + component["Heat Rate Incr"] = Quantity(fn.proportional_term) + fuel_cost = fuel_curve.fuel_cost + if fuel_cost: + component["fuel_price"] = Quantity(fuel_cost) + elif isinstance(op_cost, RenewableGenerationCost): + logger.info(f"No fuel-related data for renewable generator={component.get("name")}") + elif isinstance(op_cost, HydroGenerationCost): + logger.info(f"No heat rate or fuel data for hydro generator={component.get("name")}") + else: + logger.warning(f"Missing operation cost for generator={component.get("name")}") + return component + def get_valid_records_properties( self, component_list, @@ -179,6 +215,8 @@ def get_valid_records_properties( component["Heat Rate"] = ext_dict["heat_rate"] if ext_dict.get("fuel_price"): component["Fuel Price"] = ext_dict["fuel_price"] + # if "operation_cost" in component: + # component = self.parse_operation_cost(component) for property_name, property_value in component.items(): if valid_properties is not None: if property_name in valid_properties: diff --git a/src/r2x/exporter/plexos.py b/src/r2x/exporter/plexos.py index d4b4fe9e..6036d2f9 100644 --- a/src/r2x/exporter/plexos.py +++ b/src/r2x/exporter/plexos.py @@ -79,7 +79,7 @@ def run(self, *args, new_database: bool = True, **kwargs) -> "PlexosExporter": """Run the exporter.""" logger.info("Starting {}", self.__class__.__name__) - self.export_data_files() + # self.export_data_files() # If starting w/o a reference file we add our custom models and objects if new_database: @@ -106,6 +106,10 @@ def _get_time_series_properties(self, component): # noqa: C901 if not self.system.has_time_series(component): return + if len(self.system.list_time_series(component)) > 1: + # NOTE:@pedro this is a temporary fix for the multiple time series issue. + return + ts_metadata = self.system.get_time_series(component) config_dict = self.config.__dict__ @@ -173,7 +177,7 @@ def insert_component_properties( scenario = scenario or self.plexos_scenario if not records: records = [ - component.model_dump(exclude_none=True, exclude=exclude_fields) + component.model_dump(exclude_none=True, exclude=exclude_fields, mode="python") for component in self.system.get_components(component_type, filter_func=filter_func) ] @@ -542,6 +546,8 @@ def add_reserves(self) -> None: exclude_none=True, exclude=[*NESTED_ATTRIBUTES, "max_requirement"] ) + if not reserve.region: + return regions = self.system.get_components( ACBus, filter_func=lambda x: x.load_zone.name == reserve.region.name ) diff --git a/src/r2x/models/costs.py b/src/r2x/models/costs.py index 2eceb480..bf7edc80 100644 --- a/src/r2x/models/costs.py +++ b/src/r2x/models/costs.py @@ -1,13 +1,15 @@ """Cost related functions.""" -from infrasys import Component +# from infrasys import Component +from infrasys.models import InfraSysBaseModelWithIdentifers + from typing import Annotated from pydantic import Field from infrasys.cost_curves import ProductionVariableCostCurve from r2x.units import Currency, FuelPrice -class OperationalCost(Component): +class OperationalCost(InfraSysBaseModelWithIdentifers): name: Annotated[str, Field(frozen=True)] = "" diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index d4ba9902..bc793058 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -140,7 +140,7 @@ def __init__(self, *args, xml_file: str | None = None, **kwargs) -> None: assert self.config.run_folder self.run_folder = Path(self.config.run_folder) self.system = System(name=self.config.name, auto_add_composed_components=True) - self.property_map = self.config.defaults["plexos_property_map"] + self.property_map = self.config.defaults["plexos_input_property_map"] self.device_map = self.config.defaults["plexos_device_map"] self.fuel_map = self.config.defaults["plexos_fuel_map"] self.device_match_string = self.config.defaults["device_name_inference_map"] @@ -892,7 +892,7 @@ def _construct_value_curves(self, mapped_records, generator_name): if heat_rate_avg: fn = LinearFunctionData(proportional_term=heat_rate_avg.magnitude, constant_term=0) vc = AverageRateCurve( - name=f"{generator_name}_HR", + # name=f"{generator_name}_HR", function_data=fn, initial_input=heat_rate_avg.magnitude, ) @@ -910,7 +910,10 @@ def _construct_value_curves(self, mapped_records, generator_name): logger.warning("Heat Rate type not implemented for generator={}", generator_name) fn = None if not vc: - vc = InputOutputCurve(name=f"{generator_name}_HR", function_data=fn) + vc = InputOutputCurve( + # name=f"{generator_name}_HR", + function_data=fn + ) mapped_records["hr_value_curve"] = vc return mapped_records From 4c6aa7ee8cfad73920a7c4fb239aa689f2580fd6 Mon Sep 17 00:00:00 2001 From: pesap Date: Wed, 25 Sep 2024 13:26:37 -0600 Subject: [PATCH 05/11] fix: Updated compatibility with infrasys List of changes: - Added new function called `haskey` for dictionaries with testing, - Updated cost function exporting to match changes on infrasys, - Updated pyproject to include new test marks, - Updated model to include operational cost --- pyproject.toml | 1 + src/r2x/api.py | 95 +++++++++++++++++++++--------------- src/r2x/models/core.py | 3 +- src/r2x/models/costs.py | 20 ++++++-- src/r2x/models/generators.py | 6 +-- src/r2x/utils.py | 10 ++++ tests/test_utils.py | 18 +++++++ 7 files changed, 104 insertions(+), 49 deletions(-) create mode 100644 tests/test_utils.py diff --git a/pyproject.toml b/pyproject.toml index 96a1069f..07d4d7ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,6 +146,7 @@ pythonpath = [ markers = [ "exporters: Tests related to exporters", "plexos: Tests related to plexos", + "utils: Util functions" ] [tool.coverage.run] diff --git a/src/r2x/api.py b/src/r2x/api.py index 9ed78776..f46949b2 100644 --- a/src/r2x/api.py +++ b/src/r2x/api.py @@ -7,13 +7,14 @@ from pathlib import Path from itertools import chain from collections.abc import Iterable +from types import NotImplementedType from loguru import logger from infrasys.component import Component from infrasys.system import System as ISSystem + +from r2x.utils import haskey from .__version__ import __data_model_version__ -import uuid -import infrasys.cost_curves class System(ISSystem): @@ -52,7 +53,11 @@ def export_component_to_csv( # Get desired components to offload to csv components = map( lambda component: component.model_dump( - exclude={}, exclude_none=True, mode="json", context={"magnitude_only": True} + exclude={}, + exclude_none=True, + mode="json", + context={"magnitude_only": True}, + serialize_as_any=True, ), self.get_components(component, filter_func=filter_func), ) @@ -84,46 +89,58 @@ def _add_operation_cost_data( # noqa: C901 assert ( "uuid" in cost_field_value.keys() ), f"Operation cost field {cost_field_key} was assumed to be a component but is not." - variable_cost = self.get_component_by_uuid(uuid.UUID(cost_field_value["uuid"])) - sub_dict["variable_cost"] = variable_cost.vom_cost.function_data.proportional_term - if "fuel_cost" in variable_cost.model_fields: + variable_cost = ( + cost_field_value # self.get_component_by_uuid(uuid.UUID(cost_field_value["uuid"])) + ) + if haskey(variable_cost, ["vom_cost", "function_data"]): + sub_dict["variable_cost"] = variable_cost["vom_cost"]["function_data"][ + "proportional_term" + ] + + if "fuel_cost" in variable_cost.keys(): + assert variable_cost["fuel_cost"] is not None # Note: We multiply the fuel price by 1000 to offset the division # done by Sienna when it parses .csv files - sub_dict["fuel_price"] = variable_cost.fuel_cost * 1000 + sub_dict["fuel_price"] = variable_cost["fuel_cost"] * 1000 operation_cost_fields.add("fuel_price") - function_data = variable_cost.value_curve.function_data - if "constant_term" in function_data.model_fields: - sub_dict["heat_rate_a0"] = function_data.constant_term - operation_cost_fields.add("heat_rate_a0") - if "proportional_term" in function_data.model_fields: - sub_dict["heat_rate_a1"] = function_data.proportional_term - operation_cost_fields.add("heat_rate_a1") - if "quadratic_term" in function_data.model_fields: - sub_dict["heat_rate_a2"] = function_data.quadratic_term - operation_cost_fields.add("heat_rate_a2") - if "x_coords" in function_data.model_fields: - x_y_coords = dict(zip(function_data.x_coords, function_data.y_coords)) - match type(variable_cost): - case infrasys.cost_curves.CostCurve: - for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): - output_point_col = f"output_point_{i}" - sub_dict[output_point_col] = x_coord - operation_cost_fields.add(output_point_col) - - cost_point_col = f"cost_point_{i}" - sub_dict[cost_point_col] = y_coord - operation_cost_fields.add(cost_point_col) - - case infrasys.cost_curves.FuelCurve: - for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): - output_point_col = f"output_point_{i}" - sub_dict[output_point_col] = x_coord - operation_cost_fields.add(output_point_col) - - heat_rate_col = "heat_rate_avg_0" if i == 0 else f"heat_rate_incr_{i}" - sub_dict[heat_rate_col] = y_coord - operation_cost_fields.add(heat_rate_col) + if haskey(variable_cost, ["value_curve", "function_data"]): + function_data = variable_cost["value_curve"]["function_data"] + if "constant_term" in function_data.keys(): + sub_dict["heat_rate_a0"] = function_data["constant_term"] + operation_cost_fields.add("heat_rate_a0") + if "proportional_term" in function_data.keys(): + sub_dict["heat_rate_a1"] = function_data["proportional_term"] + operation_cost_fields.add("heat_rate_a1") + if "quadratic_term" in function_data.keys(): + sub_dict["heat_rate_a2"] = function_data["quadratic_term"] + operation_cost_fields.add("heat_rate_a2") + if "x_coords" in function_data.keys(): + x_y_coords = dict(zip(function_data.x_coords, function_data.y_coords)) + match variable_cost["variable_type"]: + case "CostCurve": + for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): + output_point_col = f"output_point_{i}" + sub_dict[output_point_col] = x_coord + operation_cost_fields.add(output_point_col) + + cost_point_col = f"cost_point_{i}" + sub_dict[cost_point_col] = y_coord + operation_cost_fields.add(cost_point_col) + + case "FuelCurve": + for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): + output_point_col = f"output_point_{i}" + sub_dict[output_point_col] = x_coord + operation_cost_fields.add(output_point_col) + + heat_rate_col = "heat_rate_avg_0" if i == 0 else f"heat_rate_incr_{i}" + sub_dict[heat_rate_col] = y_coord + operation_cost_fields.add(heat_rate_col) + case _: + raise NotImplementedType( + f"Type {variable_cost['variable_type']} variable curve not supported" + ) elif cost_field_key not in sub_dict.keys(): sub_dict[cost_field_key] = cost_field_value operation_cost_fields.add(cost_field_key) diff --git a/src/r2x/models/core.py b/src/r2x/models/core.py index 76489df0..f995a7ea 100644 --- a/src/r2x/models/core.py +++ b/src/r2x/models/core.py @@ -4,7 +4,7 @@ from infrasys.component import Component from typing import Annotated -from pydantic import Field, field_serializer +from pydantic import Field, computed_field, field_serializer from r2x.units import ureg @@ -16,6 +16,7 @@ class BaseComponent(Component): ext: dict = Field(default_factory=dict, description="Additional information of the component.") @property + @computed_field def class_type(self) -> str: """Create attribute that holds the class name.""" return type(self).__name__ diff --git a/src/r2x/models/costs.py b/src/r2x/models/costs.py index bf7edc80..4fc7cf77 100644 --- a/src/r2x/models/costs.py +++ b/src/r2x/models/costs.py @@ -1,16 +1,26 @@ """Cost related functions.""" -# from infrasys import Component -from infrasys.models import InfraSysBaseModelWithIdentifers - from typing import Annotated -from pydantic import Field +from infrasys.models import InfraSysBaseModelWithIdentifers +from pydantic import Field, computed_field from infrasys.cost_curves import ProductionVariableCostCurve from r2x.units import Currency, FuelPrice class OperationalCost(InfraSysBaseModelWithIdentifers): - name: Annotated[str, Field(frozen=True)] = "" + @property + @computed_field + def class_type(self) -> str: + """Create attribute that holds the class name.""" + return type(self).__name__ + + @property + @computed_field + def variable_type(self) -> str | None: + """Create attribute that holds the class name.""" + if not getattr(self, "variable"): + return None + return type(getattr(self, "variable")).__name__ class RenewableGenerationCost(OperationalCost): diff --git a/src/r2x/models/generators.py b/src/r2x/models/generators.py index 6e0bc52b..bcd93b07 100644 --- a/src/r2x/models/generators.py +++ b/src/r2x/models/generators.py @@ -5,7 +5,7 @@ from pydantic import Field, NonNegativeFloat from r2x.models.core import Device -from r2x.models.costs import HydroGenerationCost, RenewableGenerationCost, ThermalGenerationCost, StorageCost +from r2x.models.costs import OperationalCost from r2x.models.topology import ACBus from r2x.models.load import PowerLoad from r2x.enums import PrimeMoversType @@ -38,9 +38,7 @@ class Generator(Device): ), ), ] - operation_cost: ( - ThermalGenerationCost | RenewableGenerationCost | HydroGenerationCost | StorageCost | None - ) = None + operation_cost: OperationalCost | None = None base_power: Annotated[ ApparentPower | None, Field( diff --git a/src/r2x/utils.py b/src/r2x/utils.py index c96f2af9..ef5e0796 100644 --- a/src/r2x/utils.py +++ b/src/r2x/utils.py @@ -5,6 +5,7 @@ import json import ast from operator import attrgetter +import functools # Standard packages import os @@ -710,5 +711,14 @@ def get_property_magnitude(property_value, to_unit: str | None = None) -> float: return property_value.magnitude +def haskey(base_dict: dict, path: list[str]) -> bool: + """Return True if the dictionary has the key for the given path.""" + try: + functools.reduce(lambda x, y: x[y], path, base_dict) + return True + except (KeyError, TypeError): + return False + + DEFAULT_COLUMN_MAP = read_json("r2x/defaults/config.json").get("default_column_mapping") mapping_schema = json.loads(files("r2x.defaults").joinpath("mapping_schema.json").read_text()) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..31da9794 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,18 @@ +import pytest +from r2x.utils import haskey + + +@pytest.mark.utils +@pytest.mark.parametrize( + "base_dict, path, expected", + [ + ({"a": {"b": {"c": 1}}}, ["a", "b", "c"], True), # valid path + ({"a": {"b": {"c": 1}}}, ["a", "b"], True), # valid partial path + ({"a": {"b": {"c": 1}}}, ["a", "x"], False), # invalid key in path + ({"a": {"b": {"c": 1}}}, ["a", "b", "d"], False), # invalid path at last level + ({}, ["a"], False), # empty dictionary + ({"a": {"b": {"c": 1}}}, ["a", "b", "c", "d"], False), # path too long + ], +) +def test_haskey(base_dict, path, expected): + assert haskey(base_dict, path) == expected From bc0679a345647a91edd092b1f2621a7625657c71 Mon Sep 17 00:00:00 2001 From: pesap Date: Wed, 25 Sep 2024 13:33:51 -0600 Subject: [PATCH 06/11] fix: trailing space --- src/r2x/exporter/handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index bf910ea0..cb119697 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -189,11 +189,11 @@ def parse_operation_cost(self, component): if fuel_cost: component["fuel_price"] = Quantity(fuel_cost) elif isinstance(op_cost, RenewableGenerationCost): - logger.info(f"No fuel-related data for renewable generator={component.get("name")}") + logger.info(f"No fuel-related data for renewable generator={component.get('name')}") elif isinstance(op_cost, HydroGenerationCost): - logger.info(f"No heat rate or fuel data for hydro generator={component.get("name")}") + logger.info(f"No heat rate or fuel data for hydro generator={component.get('name')}") else: - logger.warning(f"Missing operation cost for generator={component.get("name")}") + logger.warning(f"Missing operation cost for generator={component.get('name')}") return component def get_valid_records_properties( From 1e2993eb59ab9b5c09ffea752106d09a9902146d Mon Sep 17 00:00:00 2001 From: pesap Date: Wed, 25 Sep 2024 13:37:13 -0600 Subject: [PATCH 07/11] fix: bring back time series --- src/r2x/exporter/plexos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/r2x/exporter/plexos.py b/src/r2x/exporter/plexos.py index 6036d2f9..5faeecae 100644 --- a/src/r2x/exporter/plexos.py +++ b/src/r2x/exporter/plexos.py @@ -79,7 +79,7 @@ def run(self, *args, new_database: bool = True, **kwargs) -> "PlexosExporter": """Run the exporter.""" logger.info("Starting {}", self.__class__.__name__) - # self.export_data_files() + self.export_data_files() # If starting w/o a reference file we add our custom models and objects if new_database: From 062576e6cdafa09c08cb13d792d40c98d57edcf6 Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 27 Sep 2024 11:48:21 -0600 Subject: [PATCH 08/11] fix: Added new functions for the exporters and cleaned plexos export --- pyproject.toml | 1 + src/r2x/api.py | 2 +- src/r2x/exceptions.py | 4 + src/r2x/exporter/handler.py | 131 +++++++++++++------------ src/r2x/exporter/plexos.py | 119 ++++++++++++++--------- src/r2x/exporter/utils.py | 172 +++++++++++++++++++++++++++++++++ src/r2x/models/costs.py | 4 +- src/r2x/parser/reeds.py | 4 +- tests/test_exporter_handler.py | 0 tests/test_exporter_utils.py | 76 +++++++++++++++ 10 files changed, 400 insertions(+), 113 deletions(-) create mode 100644 tests/test_exporter_handler.py create mode 100644 tests/test_exporter_utils.py diff --git a/pyproject.toml b/pyproject.toml index 07d4d7ae..a93d7859 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,7 @@ pythonpath = [ ] markers = [ "exporters: Tests related to exporters", + "exporter_utils: Tests related to exporters utils", "plexos: Tests related to plexos", "utils: Util functions" ] diff --git a/src/r2x/api.py b/src/r2x/api.py index f46949b2..824ab194 100644 --- a/src/r2x/api.py +++ b/src/r2x/api.py @@ -57,7 +57,7 @@ def export_component_to_csv( exclude_none=True, mode="json", context={"magnitude_only": True}, - serialize_as_any=True, + # serialize_as_any=True, ), self.get_components(component, filter_func=filter_func), ) diff --git a/src/r2x/exceptions.py b/src/r2x/exceptions.py index ecdeaaca..d098812a 100644 --- a/src/r2x/exceptions.py +++ b/src/r2x/exceptions.py @@ -28,3 +28,7 @@ class MultipleFilesError(Exception): class ParserError(Exception): pass + + +class FieldRemovalError(Exception): + pass diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index cb119697..212e4a0f 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -7,23 +7,15 @@ from pathlib import Path from typing import Any -import pint -from pint import Quantity import pandas as pd import numpy as np import infrasys from loguru import logger -from infrasys.base_quantity import BaseQuantity -from infrasys.function_data import ( - LinearFunctionData, - QuadraticFunctionData, -) -from infrasys.value_curves import InputOutputCurve, AverageRateCurve from r2x.api import System from r2x.config import Scenario +from r2x.exporter.utils import get_property_magnitude, modify_components from r2x.parser.handler import file_handler -from r2x.models.costs import ThermalGenerationCost, RenewableGenerationCost, HydroGenerationCost OUTPUT_FNAME = "{self.weather_year}" @@ -167,35 +159,6 @@ def export_data_files(self, time_series_folder: str = "Data") -> None: return - def parse_operation_cost(self, component): - """Parse Infrasys Operation Cost into Plexos Records.""" - op_cost = component.get("operation_cost") - if isinstance(op_cost, ThermalGenerationCost): - fuel_curve = op_cost.variable - if fuel_curve: - hr_curve = fuel_curve.value_curve - if isinstance(hr_curve, AverageRateCurve): - component["Heat Rate"] = Quantity(hr_curve.function_data.proportional_term) - elif isinstance(hr_curve, InputOutputCurve): - fn = hr_curve.function_data - if isinstance(fn, QuadraticFunctionData): - component["Heat Rate Base"] = Quantity(fn.constant_term) - component["Heat Rate Incr"] = Quantity(fn.proportional_term) - component["Heat Rate Incr2"] = Quantity(fn.quadratic_term) - elif isinstance(fn, LinearFunctionData): - component["Heat Rate Base"] = Quantity(fn.constant_term) - component["Heat Rate Incr"] = Quantity(fn.proportional_term) - fuel_cost = fuel_curve.fuel_cost - if fuel_cost: - component["fuel_price"] = Quantity(fuel_cost) - elif isinstance(op_cost, RenewableGenerationCost): - logger.info(f"No fuel-related data for renewable generator={component.get('name')}") - elif isinstance(op_cost, HydroGenerationCost): - logger.info(f"No heat rate or fuel data for hydro generator={component.get('name')}") - else: - logger.warning(f"Missing operation cost for generator={component.get('name')}") - return component - def get_valid_records_properties( self, component_list, @@ -205,51 +168,95 @@ def get_valid_records_properties( ): """Return a validadted list of properties to the given property_map.""" result = [] - component_list_mapped = [ - {property_map.get(key, key): value for key, value in d.items()} for d in component_list - ] - for component in component_list_mapped: + for component in component_list: component_dict = {"name": component["name"]} # We need the name to match it with the membership. if ext_dict := component.get("ext"): if ext_dict.get("heat_rate"): component["Heat Rate"] = ext_dict["heat_rate"] if ext_dict.get("fuel_price"): component["Fuel Price"] = ext_dict["fuel_price"] - # if "operation_cost" in component: - # component = self.parse_operation_cost(component) + if "operation_cost" in component: + pass for property_name, property_value in component.items(): if valid_properties is not None: if property_name in valid_properties: - property_value = self.get_property_magnitude( + property_value = get_property_magnitude( property_value, to_unit=unit_map.get(property_name) ) component_dict[property_name] = property_value else: - property_value = self.get_property_magnitude( + property_value = get_property_magnitude( property_value, to_unit=unit_map.get(property_name) ) component_dict[property_name] = property_value result.append(component_dict) return result - def get_property_magnitude(self, property_value, to_unit: str | None = None) -> float: - """Return magnitude with the given units for a pint Quantity. - Parameters - ---------- - property_name +def get_export_records(component_list: list[dict[str, Any]], *update_funcs: Callable) -> list[dict[str, Any]]: + """Apply update functions to a list of components and return the modified list. - property_value - pint.Quantity to extract magnitude from - to_unit - String that contains the unit conversion desired. Unit must be compatible. - """ - if not isinstance(property_value, pint.Quantity | BaseQuantity): - return property_value - if to_unit: - unit = to_unit.replace("$", "usd") # Dollars are named usd on pint - property_value = property_value.to(unit) - return property_value.magnitude + Parameters + ---------- + component_list : list[dict[str, Any]] + A list of dictionaries representing components to be updated. + *update_funcs : Callable + Variable number of update functions to be applied to each component. + + Returns + ------- + list[dict[str, Any]] + A list of updated component dictionaries. + + Examples + -------- + >>> def update_name(component): + ... component["name"] = component["name"].upper() + ... return component + >>> def add_prefix(component): + ... component["id"] = f"PREFIX_{component['id']}" + ... return component + >>> components = [{"id": "001", "name": "Component A"}, {"id": "002", "name": "Component B"}] + >>> updated_components = get_export_records(components, update_name, add_prefix) + >>> updated_components + [{'id': 'PREFIX_001', 'name': 'COMPONENT A'}, {'id': 'PREFIX_002', 'name': 'COMPONENT B'}] + """ + update_functions = modify_components(*update_funcs) + return [update_functions(component) for component in component_list] + + +def get_export_properties(component, *update_funcs: Callable) -> dict[str, Any]: + """Apply update functions to a single component and return the modified component. + + Parameters + ---------- + component : dict[str, Any] + A dictionary representing a component to be updated. + *update_funcs : Callable + Variable number of update functions to be applied to the component. + + Returns + ------- + dict[str, Any] + The updated component dictionary. + + Examples + -------- + >>> def update_status(component): + ... component["status"] = "active" + ... return component + >>> def add_timestamp(component): + ... from datetime import datetime + ... + ... component["last_updated"] = datetime.now().isoformat() + ... return component + >>> component = {"id": "003", "name": "Component C"} + >>> updated_component = get_export_properties(component, update_status, add_timestamp) + >>> updated_component + {'id': '003', 'name': 'Component C', 'status': 'active', 'last_updated': '2024-09-27T10:30'} + """ + update_functions = modify_components(*update_funcs) + return update_functions(component) def get_exporter( diff --git a/src/r2x/exporter/plexos.py b/src/r2x/exporter/plexos.py index 5faeecae..1022ffa4 100644 --- a/src/r2x/exporter/plexos.py +++ b/src/r2x/exporter/plexos.py @@ -1,20 +1,27 @@ """Create PLEXOS model from translated ReEDS data.""" from argparse import ArgumentParser +from functools import partial from importlib.resources import files from typing import Any import uuid import string from collections.abc import Callable + from infrasys.component import Component from loguru import logger from r2x.enums import ReserveType -from r2x.exporter.handler import BaseExporter +from r2x.exporter.handler import BaseExporter, get_export_properties, get_export_records from plexosdb import PlexosSQLite from plexosdb.enums import ClassEnum, CollectionEnum -from r2x.exporter.utils import get_reserve_type +from r2x.exporter.utils import ( + apply_pint_deconstruction, + apply_property_map, + apply_valid_properties, + get_reserve_type, +) from r2x.models import ( ACBus, Emission, @@ -35,7 +42,7 @@ TransmissionInterface, ) from r2x.units import get_magnitude -from r2x.utils import custom_attrgetter, get_enum_from_string, read_json, get_property_magnitude +from r2x.utils import custom_attrgetter, get_enum_from_string, read_json NESTED_ATTRIBUTES = ["ext", "bus", "services"] TIME_SERIES_PROPERTIES = ["Min Provision", "Static Risk"] @@ -177,7 +184,9 @@ def insert_component_properties( scenario = scenario or self.plexos_scenario if not records: records = [ - component.model_dump(exclude_none=True, exclude=exclude_fields, mode="python") + component.model_dump( + exclude_none=True, exclude=exclude_fields, mode="python", serialize_as_any=True + ) for component in self.system.get_components(component_type, filter_func=filter_func) ] @@ -195,14 +204,16 @@ def insert_component_properties( case _: custom_map = {} property_map = self.property_map | custom_map - valid_component_properties = self.get_valid_records_properties( + + export_records = get_export_records( records, - property_map, - self.default_units, - valid_properties=collection_properties, + partial(apply_operation_cost), + partial(apply_property_map, property_map=property_map), + partial(apply_pint_deconstruction, unit_map=self.default_units), + partial(apply_valid_properties, valid_properties=collection_properties, add_name=True), ) self._db_mgr.add_property_from_records( - valid_component_properties, + export_records, parent_class=parent_class, parent_object_name=parent_object_name, collection=collection, @@ -401,14 +412,16 @@ def add_lines(self) -> None: MonitoredLine, parent_class=ClassEnum.System, collection=CollectionEnum.Lines ) - for line in self.system.get_components(MonitoredLine): - properties = self.get_valid_component_properties( + # Add additional properties if any and membersips + collection_properties = self._db_mgr.get_valid_properties( + collection=CollectionEnum.Lines, parent_class=ClassEnum.System, child_class=ClassEnum.Line + ) + for line in self.system.get_components(MonitoredLine, filter_func=lambda x: getattr(x, "ext", False)): + properties = get_export_properties( line.ext, - property_map=self.property_map, - unit_map=self.default_units, - collection=CollectionEnum.Lines, - parent_class=ClassEnum.System, - child_class=ClassEnum.Line, + partial(apply_property_map, property_map=self.property_map), + partial(apply_pint_deconstruction, unit_map=self.default_units), + partial(apply_valid_properties, valid_properties=collection_properties), ) for property_name, property_value in properties.items(): self._db_mgr.add_property( @@ -552,6 +565,11 @@ def add_reserves(self) -> None: ACBus, filter_func=lambda x: x.load_zone.name == reserve.region.name ) + collection_properties = self._db_mgr.get_valid_properties( + collection=CollectionEnum.Regions, + parent_class=ClassEnum.Reserve, + child_class=ClassEnum.Region, + ) for region in regions: self._db_mgr.add_membership( reserve.name, @@ -560,13 +578,11 @@ def add_reserves(self) -> None: child_class=ClassEnum.Region, collection=CollectionEnum.Regions, ) - properties = self.get_valid_component_properties( + properties = get_export_properties( component_dict, - property_map=self.property_map, - unit_map=self.default_units, - collection=CollectionEnum.Regions, - parent_class=ClassEnum.Reserve, - child_class=ClassEnum.Region, + partial(apply_property_map, property_map=self.property_map), + partial(apply_pint_deconstruction, unit_map=self.default_units), + partial(apply_valid_properties, valid_properties=collection_properties), ) if properties: for property_name, property_value in properties.items(): @@ -862,27 +878,40 @@ def _get_category_id(self, component, category_attribute, categories_ids, defaul return categories_ids[default_category] return categories_ids[category_to_get] - def get_valid_component_properties( - self, - component_dict: dict, - property_map: dict[str, str], - unit_map: dict[str, str], - collection: CollectionEnum, - parent_class: ClassEnum | None = None, - child_class: ClassEnum | None = None, - ): - """Validate single component properties.""" - valid_component_properties = {} - component_dict_mapped = {property_map.get(key, key): value for key, value in component_dict.items()} - # collection_properties = self._db_mgr.query( - # f"select name, property_id from t_property where collection_id={collection}" - # ) - valid_collection_properties = self._db_mgr.get_valid_properties( - collection, parent_class=parent_class, child_class=child_class - ) - for property_name, property_value in component_dict_mapped.items(): - if property_name in valid_collection_properties: - property_value = get_property_magnitude(property_value, to_unit=unit_map.get(property_name)) - valid_component_properties[property_name] = property_value - return valid_component_properties +def apply_operation_cost(component: dict) -> dict[str, Any]: + """Parse Infrasys Operation Cost into Plexos Records.""" + if not (cost := component.get("operation_cost")): + return component + match cost["class_type"]: + case "ThermalGenerationCost": + if shut_down := cost.get("start_up"): + component["Start Cost"] = shut_down + if shut_down := cost.get("shut_down"): + component["Shutdown Cost"] = shut_down + + if cost.get("variable"): + component = _variable_type_parsing(component, cost) + case _: + pass + return component + + +def _variable_type_parsing(component: dict, cost_dict: dict[str, Any]) -> dict[str, Any]: + fuel_curve = cost_dict["variable"] + variable_type = cost_dict["variable_type"] + function_data = fuel_curve["value_curve"]["function_data"] + match variable_type: + case "FuelCurve": + match fuel_curve["value_curve_type"]: + case "AverageRateCurve": + component["Heat Rate"] = function_data["proportional_term"] + case "InputOutputCurve": + raise NotImplementedError("`InputOutputCurve` not yet implemented on Plexos exporter.") + if fuel_cost := fuel_curve.get("fuel_cost"): + component["Fuel Price"] = fuel_cost + if vom_cost := fuel_curve.get("vom_cost"): + component["VO&M Charge"] = vom_cost["function_data"]["proportional_term"] + case "CostCurve": + raise NotImplementedError("`CostCurve` operational cost not yet implemented on Plexos exporter.") + return component diff --git a/src/r2x/exporter/utils.py b/src/r2x/exporter/utils.py index 69c2b8f7..30f22be8 100644 --- a/src/r2x/exporter/utils.py +++ b/src/r2x/exporter/utils.py @@ -1,6 +1,13 @@ """Helper functions for the exporters.""" +from typing import Any +from collections.abc import Callable +import copy +from functools import wraps from r2x.enums import ReserveType, ReserveDirection +from r2x.exceptions import FieldRemovalError +import pint +from infrasys.base_quantity import BaseQuantity def get_reserve_type( @@ -18,3 +25,168 @@ def get_reserve_type( ReserveDirection[reserve_types["default"]["direction"]], reserve_types, ) + + +def required_fields(*fields: str | list[str] | set): + """Specify required fields for the transformation.""" + + def decorator( + func: Callable, + ) -> Callable: + @wraps(func) + def wrapper(component_data, *args, **kwargs): + original_data = copy.deepcopy(component_data) + result = func(component_data) + removed_fields = set(original_data.keys()) - set(result.keys()) + if removed_fields & set(fields): + removed_required = removed_fields & set(fields) + raise FieldRemovalError( + f"Transformation {func.__name__} removed required fields: {removed_required}" + ) + return result + + return wrapper + + return decorator + + +def compose( + *functions: Callable[[dict[str, Any]], dict[str, Any]], +) -> Callable[[dict[str, Any]], dict[str, Any]]: + """ + Compose multiple functions, applying them sequentially to the input data. + + Parameters + ---------- + functions : Callable[[Dict[str, Any]], Dict[str, Any]] + Functions that transform a dictionary. + + Returns + ------- + Callable[[Dict[str, Any]], Dict[str, Any]] + A function that sequentially applies the provided functions to a dictionary. + """ + + def apply_all(data: dict[str, Any]) -> dict[str, Any]: + for func in functions: + data = func(data) + return data + + return apply_all + + +def modify_components( + *transform_functions: Callable[[dict[str, Any]], dict[str, Any]], +) -> Callable[[dict[str, Any]], dict[str, Any]]: + """Apply multiple transformations to components.""" + return compose(*transform_functions) + + +def apply_property_map(component: dict[str, Any], property_map: dict[str, str]) -> dict[str, Any]: + """Apply a key mapping to component keys. + + For each key in the `component` dictionary, if the key exists in the `property_map`, + it will be replaced by the corresponding mapped key. Otherwise, the original key is used. + + Parameters + ---------- + component : dict[str, Any] + The original dictionary where keys represent component properties. + property_map : dict[str, str] + A dictionary mapping old property names to new property names. + + Returns + ------- + dict[str, Any] + A new dictionary where the keys have been remapped according to `property_map`. + + Examples + -------- + >>> component = {"voltage": 230, "current": 10} + >>> property_map = {"voltage": "v", "current": "i"} + >>> apply_property_map(component, property_map) + {'v': 230, 'i': 10} + + >>> component = {"voltage": 230, "resistance": 50} + >>> property_map = {"voltage": "v", "current": "i"} + >>> apply_property_map(component, property_map) + {'v': 230, 'resistance': 50} + """ + return {property_map.get(key, key): value for key, value in component.items()} + + +def apply_pint_deconstruction(component: dict[str, Any], unit_map: dict[str, str]) -> dict[str, Any]: + """Get property magnitude from a pint Quantity. + + If unit_map is passed, convert to units specified + + Parameters + ---------- + component: dict[str, Any] + Dictionary representation of component. Typically created with `.model_dump()` + unit_map: dict[str, str] + Map to convert property to a desired units. Optional. + """ + return { + property: get_property_magnitude(property_value, to_unit=unit_map.get(property, None)) + for property, property_value in component.items() + } + + +def apply_valid_properties( + component: dict[str, Any], valid_properties: list[str], add_name: bool = False +) -> dict[str, Any]: + """Filter a component dictionary to only include keys that are in the valid properties list. + + Parameters + ---------- + component : dict of str to Any + A dictionary representing the component with properties as keys. + valid_properties : list of str + A list of valid property names. Only keys present in this list will be kept in the output. + + Returns + ------- + dict + A new dictionary with only the properties from `valid_properties`. + + Examples + -------- + >>> component = {"voltage": 230, "current": 10, "resistance": 50} + >>> valid_properties = ["voltage", "current"] + >>> apply_valid_properties(component, valid_properties) + {'voltage': 230, 'current': 10} + """ + if add_name: + default_properties = ["name"] + return { + key: value + for key, value in component.items() + if key in valid_properties or key in default_properties + } + return {key: value for key, value in component.items() if key in valid_properties} + + +def get_property_magnitude(property_value, to_unit: str | None = None) -> Any: + """Return magnitude with the given units for a pint Quantity. + + Parameters + ---------- + property_name + + property_value + pint.Quantity to extract magnitude from + to_unit + String that contains the unit conversion desired. Unit must be compatible. + + Returns + ------- + float + Magnitude representation of the `pint.Quantity` or original value. + """ + if not isinstance(property_value, pint.Quantity | BaseQuantity): + return property_value + if to_unit: + unit = to_unit.replace("$", "usd") # Dollars are named usd on pint + property_value = property_value.to(unit) + return property_value.magnitude diff --git a/src/r2x/models/costs.py b/src/r2x/models/costs.py index 4fc7cf77..ba687cc5 100644 --- a/src/r2x/models/costs.py +++ b/src/r2x/models/costs.py @@ -8,14 +8,14 @@ class OperationalCost(InfraSysBaseModelWithIdentifers): + @computed_field # type: ignore[prop-decorator] @property - @computed_field def class_type(self) -> str: """Create attribute that holds the class name.""" return type(self).__name__ + @computed_field # type: ignore[prop-decorator] @property - @computed_field def variable_type(self) -> str | None: """Create attribute that holds the class name.""" if not getattr(self, "variable"): diff --git a/src/r2x/parser/reeds.py b/src/r2x/parser/reeds.py index 062336b9..574dfeea 100644 --- a/src/r2x/parser/reeds.py +++ b/src/r2x/parser/reeds.py @@ -433,7 +433,7 @@ def _construct_generators(self) -> None: # noqa: C901 if heat_rate := row.get("heat_rate"): heat_rate_curve = AverageRateCurve( function_data=LinearFunctionData( - proportional_term=heat_rate.magnitude, + proportional_term=heat_rate, constant_term=0, ), initial_input=heat_rate.magnitude, @@ -464,8 +464,6 @@ def _construct_generators(self) -> None: # noqa: C901 "reeds_tech": row["tech"], "reeds_vintage": row["tech_vintage"], "Commit": commit, - "heat_rate": row.get("heat_rate"), - "fuel_price": row.get("fuel_price"), } ) diff --git a/tests/test_exporter_handler.py b/tests/test_exporter_handler.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_exporter_utils.py b/tests/test_exporter_utils.py new file mode 100644 index 00000000..1c4cd3e5 --- /dev/null +++ b/tests/test_exporter_utils.py @@ -0,0 +1,76 @@ +import pytest +from pint import Quantity +from r2x.exporter.utils import ( + apply_property_map, + apply_valid_properties, + apply_pint_deconstruction, + get_property_magnitude, +) + + +@pytest.mark.exporter_utils +def test_apply_property_map(): + """Test the apply_property_map function.""" + component = {"voltage": 230, "current": 10} + property_map = {"voltage": "v", "current": "i"} + + result = apply_property_map(component, property_map) + assert result == {"v": 230, "i": 10} + + component = {"voltage": 230, "resistance": 50} + property_map = {"voltage": "v", "current": "i"} + result = apply_property_map(component, property_map) + assert result == {"v": 230, "resistance": 50} + + # Test for empty component + result = apply_property_map({}, property_map) + assert result == {} + + +@pytest.mark.exporter_utils +def test_apply_pint_deconstruction(): + """Test the apply_pint_deconstruction function.""" + component = {"length": Quantity(100, "meters"), "time": Quantity(3600, "seconds")} + unit_map = {"length": "km", "time": "h"} + + result = apply_pint_deconstruction(component, unit_map) + assert result["length"] == 0.1 # 100 meters to kilometers + assert result["time"] == 1 # 3600 seconds to hours + + result = apply_pint_deconstruction(component, {}) + assert result["length"] == 100 + assert result["time"] == 3600 + + +@pytest.mark.exporter_utils +def test_apply_valid_properties(): + """Test the apply_valid_properties function.""" + component = {"voltage": 230, "current": 10, "resistance": 50} + valid_properties = ["voltage", "current"] + + # Test filtering + result = apply_valid_properties(component, valid_properties) + assert result == {"voltage": 230, "current": 10} + + component_with_name = {"voltage": 230, "current": 10, "resistance": 50, "name": "Component A"} + valid_properties_with_name = ["voltage", "current"] + + result_with_name = apply_valid_properties(component_with_name, valid_properties_with_name, add_name=True) + assert result_with_name == {"voltage": 230, "current": 10, "name": "Component A"} + + result_empty = apply_valid_properties(component, []) + assert result_empty == {} + + +@pytest.mark.exporter_utils +def test_get_property_magnitude(): + """Test the get_property_magnitude function.""" + q1 = Quantity(100, "meters") # Pint Quantity + q2 = Quantity(50, "kilograms") # Pint Quantity + q3 = 200 # Not a Quantity + + assert get_property_magnitude(q1, "kilometers") == 0.1 # Convert 100 meters to kilometers + assert get_property_magnitude(q2, "grams") == 50000 # Convert 50 kg to grams + + assert get_property_magnitude(q3) == 200 # No conversion for a non-Quantity + assert get_property_magnitude(q1) == 100 # Magnitude of Quantity without conversion From 423faab95d6af4a7d93b43910e374ca539b8085f Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 27 Sep 2024 12:02:56 -0600 Subject: [PATCH 09/11] fix: Apply sienna exporter changes --- src/r2x/exporter/handler.py | 35 +---------------------------------- src/r2x/exporter/sienna.py | 18 +++++++++--------- tests/test_sienna_exporter.py | 2 ++ 3 files changed, 12 insertions(+), 43 deletions(-) diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index 212e4a0f..46b90536 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -14,7 +14,7 @@ from r2x.api import System from r2x.config import Scenario -from r2x.exporter.utils import get_property_magnitude, modify_components +from r2x.exporter.utils import modify_components from r2x.parser.handler import file_handler OUTPUT_FNAME = "{self.weather_year}" @@ -159,39 +159,6 @@ def export_data_files(self, time_series_folder: str = "Data") -> None: return - def get_valid_records_properties( - self, - component_list, - property_map: dict[str, str], - unit_map: dict[str, str], - valid_properties: list | None = None, - ): - """Return a validadted list of properties to the given property_map.""" - result = [] - for component in component_list: - component_dict = {"name": component["name"]} # We need the name to match it with the membership. - if ext_dict := component.get("ext"): - if ext_dict.get("heat_rate"): - component["Heat Rate"] = ext_dict["heat_rate"] - if ext_dict.get("fuel_price"): - component["Fuel Price"] = ext_dict["fuel_price"] - if "operation_cost" in component: - pass - for property_name, property_value in component.items(): - if valid_properties is not None: - if property_name in valid_properties: - property_value = get_property_magnitude( - property_value, to_unit=unit_map.get(property_name) - ) - component_dict[property_name] = property_value - else: - property_value = get_property_magnitude( - property_value, to_unit=unit_map.get(property_name) - ) - component_dict[property_name] = property_value - result.append(component_dict) - return result - def get_export_records(component_list: list[dict[str, Any]], *update_funcs: Callable) -> list[dict[str, Any]]: """Apply update functions to a list of components and return the modified list. diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 57be941d..3cac628d 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -1,6 +1,7 @@ """R2X Sienna system exporter.""" # System packages +from functools import partial import json import os from typing import Any @@ -9,7 +10,8 @@ from loguru import logger # Local imports -from r2x.exporter.handler import BaseExporter +from r2x.exporter.handler import BaseExporter, get_export_records +from r2x.exporter.utils import apply_pint_deconstruction, apply_property_map from r2x.models import ( ACBranch, Bus, @@ -53,6 +55,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.output_data = {} self.property_map = self.config.defaults.get("sienna_property_map", {}) + self.unit_map = self.config.defaults.get("sienna_unit_map", {}) def run(self, *args, path=None, **kwargs) -> "SiennaExporter": """Run sienna exporter workflow. @@ -217,7 +220,6 @@ def process_gen_data(self, fname="gen.csv"): "rating", "unit_type", "active_power", - "active_power_limits_max", "min_rated_capacity", "min_down_time", "min_up_time", @@ -287,7 +289,6 @@ def process_reserves_data(self, fname="reserves.csv") -> None: output_dict = reserve output_dict["direction"] = reserve["direction"].name output_dict["eligible_device_categories"] = "(Generator,Storage)" - output_dict["requirement"] = reserve["max_requirement"] contributing_devices = reserve_map.get(reserve["name"]) output_dict["contributing_devices"] = str(tuple(contributing_devices)).replace( # type: ignore "'", "" @@ -324,7 +325,6 @@ def process_storage_data(self, fname="storage.csv") -> None: "rating", "input_efficiency", "output_efficiency", - "initial_energy", "storage_capacity", "min_storage_capacity", "max_storage_capacity", @@ -334,11 +334,11 @@ def process_storage_data(self, fname="storage.csv") -> None: "unit_type", ] - generic_storage = self.get_valid_records_properties( - self.system.to_records(Storage), - property_map=self.config.defaults["sienna_property_map"], - unit_map=self.config.defaults["sienna_unit_map"], - # valid_properties=output_fields, + generic_storage = get_export_records( + list(self.system.to_records(Storage)), + partial(apply_property_map, property_map=self.property_map), + partial(apply_pint_deconstruction, unit_map=self.unit_map), + # partial(apply_valid_properties, valid_properties=output_fields), ) hydro_pump = list(self.system.to_records(HydroPumpedStorage)) storage_list = generic_storage + hydro_pump diff --git a/tests/test_sienna_exporter.py b/tests/test_sienna_exporter.py index 5176d656..bd8c323d 100644 --- a/tests/test_sienna_exporter.py +++ b/tests/test_sienna_exporter.py @@ -26,10 +26,12 @@ def sienna_exporter(scenario_instance, infrasys_test_system, tmp_folder): return SiennaExporter(config=scenario_instance, system=infrasys_test_system, output_folder=tmp_folder) +@pytest.mark.sienna def test_sienna_exporter_instance(sienna_exporter): assert isinstance(sienna_exporter, SiennaExporter) +@pytest.mark.sienna def test_sienna_exporter_run(sienna_exporter, tmp_folder): exporter = sienna_exporter.run() From 740db027f3e7bde647e5703e437b044b6aff581a Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 27 Sep 2024 12:19:32 -0600 Subject: [PATCH 10/11] fix: added value curve_type on r2x instead of infrasys --- src/r2x/models/costs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/r2x/models/costs.py b/src/r2x/models/costs.py index ba687cc5..960d34ba 100644 --- a/src/r2x/models/costs.py +++ b/src/r2x/models/costs.py @@ -5,6 +5,7 @@ from pydantic import Field, computed_field from infrasys.cost_curves import ProductionVariableCostCurve from r2x.units import Currency, FuelPrice +from operator import attrgetter class OperationalCost(InfraSysBaseModelWithIdentifers): @@ -22,6 +23,14 @@ def variable_type(self) -> str | None: return None return type(getattr(self, "variable")).__name__ + @computed_field # type: ignore[prop-decorator] + @property + def value_curve_type(self) -> str | None: + """Create attribute that holds the class name.""" + if not attrgetter("variable.value_curve")(self): + return None + return type(attrgetter("variable.value_curve")(self)).__name__ + class RenewableGenerationCost(OperationalCost): curtailment_cost: ProductionVariableCostCurve | None = None From d7bd2924b032719eade773e192c13fcc7861a75f Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 27 Sep 2024 14:35:17 -0600 Subject: [PATCH 11/11] fix: Added new test and propagate more changes to the sienna exporter --- pyproject.toml | 1 + src/r2x/api.py | 119 ++---------------- src/r2x/defaults/sienna_config.json | 94 ++++++++++++++ src/r2x/exporter/plexos.py | 3 +- src/r2x/exporter/sienna.py | 184 +++++++++++++++++++++++----- src/r2x/exporter/utils.py | 47 +++++++ tests/test_exporter_utils.py | 54 ++++++++ tests/test_plexos_exporter.py | 4 + tests/test_sienna_exporter.py | 105 +++++++++++++++- 9 files changed, 468 insertions(+), 143 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a93d7859..7dd4390e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,6 +147,7 @@ markers = [ "exporters: Tests related to exporters", "exporter_utils: Tests related to exporters utils", "plexos: Tests related to plexos", + "sienna: Tests related to sienna", "utils: Util functions" ] diff --git a/src/r2x/api.py b/src/r2x/api.py index 6c9db9b0..86ce5bf2 100644 --- a/src/r2x/api.py +++ b/src/r2x/api.py @@ -1,19 +1,16 @@ """R2X API for data model.""" import csv -import json from collections.abc import Callable from os import PathLike from pathlib import Path -from itertools import chain from collections.abc import Iterable -from types import NotImplementedType from loguru import logger +import inspect from infrasys.component import Component from infrasys.system import System as ISSystem -from r2x.utils import haskey from .__version__ import __data_model_version__ @@ -73,123 +70,27 @@ def export_component_to_csv( **dict_writer_kwargs, ) - def _add_operation_cost_data( # noqa: C901 - self, - data: Iterable[dict], - fields: list | None = None, - ): - operation_cost_fields = set() - x_y_coords = None - for sub_dict in data: - if "operation_cost" not in sub_dict.keys(): - continue - - operation_cost = sub_dict["operation_cost"] - for cost_field_key, cost_field_value in operation_cost.items(): - if isinstance(cost_field_value, dict): - assert ( - "uuid" in cost_field_value.keys() - ), f"Operation cost field {cost_field_key} was assumed to be a component but is not." - variable_cost = ( - cost_field_value # self.get_component_by_uuid(uuid.UUID(cost_field_value["uuid"])) - ) - if haskey(variable_cost, ["vom_cost", "function_data"]): - sub_dict["variable_cost"] = variable_cost["vom_cost"]["function_data"][ - "proportional_term" - ] - - if "fuel_cost" in variable_cost.keys(): - assert variable_cost["fuel_cost"] is not None - # Note: We multiply the fuel price by 1000 to offset the division - # done by Sienna when it parses .csv files - sub_dict["fuel_price"] = variable_cost["fuel_cost"] * 1000 - operation_cost_fields.add("fuel_price") - - if haskey(variable_cost, ["value_curve", "function_data"]): - function_data = variable_cost["value_curve"]["function_data"] - if "constant_term" in function_data.keys(): - sub_dict["heat_rate_a0"] = function_data["constant_term"] - operation_cost_fields.add("heat_rate_a0") - if "proportional_term" in function_data.keys(): - sub_dict["heat_rate_a1"] = function_data["proportional_term"] - operation_cost_fields.add("heat_rate_a1") - if "quadratic_term" in function_data.keys(): - sub_dict["heat_rate_a2"] = function_data["quadratic_term"] - operation_cost_fields.add("heat_rate_a2") - if "x_coords" in function_data.keys(): - x_y_coords = dict(zip(function_data.x_coords, function_data.y_coords)) - match variable_cost["variable_type"]: - case "CostCurve": - for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): - output_point_col = f"output_point_{i}" - sub_dict[output_point_col] = x_coord - operation_cost_fields.add(output_point_col) - - cost_point_col = f"cost_point_{i}" - sub_dict[cost_point_col] = y_coord - operation_cost_fields.add(cost_point_col) - - case "FuelCurve": - for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): - output_point_col = f"output_point_{i}" - sub_dict[output_point_col] = x_coord - operation_cost_fields.add(output_point_col) - - heat_rate_col = "heat_rate_avg_0" if i == 0 else f"heat_rate_incr_{i}" - sub_dict[heat_rate_col] = y_coord - operation_cost_fields.add(heat_rate_col) - case _: - raise NotImplementedType( - f"Type {variable_cost['variable_type']} variable curve not supported" - ) - elif cost_field_key not in sub_dict.keys(): - sub_dict[cost_field_key] = cost_field_value - operation_cost_fields.add(cost_field_key) - else: - pass - - fields.remove("operation_cost") # type: ignore - fields.extend(list(operation_cost_fields)) # type: ignore - - return data, fields - def _export_dict_to_csv( self, data: Iterable[dict], fpath: PathLike, fields: list | None = None, - key_mapping: dict | None = None, - unnest_key: str = "name", + # key_mapping: dict | None = None, + # unnest_key: str = "name", **dict_writer_kwargs, ): - # Remaping keys - # NOTE: It does not work recursively for nested components - if key_mapping: - data = [ - {key_mapping.get(key, key): value for key, value in sub_dict.items()} for sub_dict in data - ] - if fields: - fields = list(map(lambda key: key_mapping.get(key, key), fields)) - - if fields is None: - fields = list(set(chain.from_iterable(data))) - - if "operation_cost" in fields: - data, fields = self._add_operation_cost_data(data, fields) + dict_writer_kwargs = { + key: value + for key, value in dict_writer_kwargs.items() + if key in inspect.getfullargspec(csv.DictWriter).args + } with open(str(fpath), "w", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fields, extrasaction="ignore", **dict_writer_kwargs) # type: ignore writer.writeheader() for row in data: - filter_row = { - key: json.dumps(value) - if key == "ext" and isinstance(value, dict) - else value - if not isinstance(value, dict) - else value.get(unnest_key) - for key, value in row.items() - } - writer.writerow(filter_row) + writer.writerow(row) + return if __name__ == "__main__": diff --git a/src/r2x/defaults/sienna_config.json b/src/r2x/defaults/sienna_config.json index dc0b3c6d..9bb6f71c 100644 --- a/src/r2x/defaults/sienna_config.json +++ b/src/r2x/defaults/sienna_config.json @@ -215,6 +215,100 @@ "ES", "FW" ], + "table_data": { + "generator": [ + "name", + "available", + "bus_id", + "fuel", + "fuel_price", + "active_power", + "reactive_power", + "active_power_limits_max", + "active_power_limits_min", + "reactive_power_limits_max", + "reactive_power_limits_min", + "min_down_time", + "min_up_time", + "ramp_limits", + "ramp_up", + "ramp_down", + "startup_heat_cold_cost", + "heat_rate_a0", + "heat_rate_a1", + "heat_rate_a2", + "heat_rate_avg_0", + "heat_rate_incr_1", + "heat_rate_incr_2", + "heat_rate_incr_3", + "heat_rate_incr_4", + "heat_rate_incr_5", + "heat_rate_incr_6", + "heat_rate_incr_7", + "heat_rate_incr_8", + "heat_rate_incr_9", + "heat_rate_incr_10", + "heat_rate_incr_11", + "heat_rate_incr_12", + "cost_point_0", + "cost_point_1", + "cost_point_2", + "cost_point_3", + "cost_point_4", + "cost_point_5", + "cost_point_6", + "cost_point_7", + "cost_point_8", + "cost_point_9", + "cost_point_10", + "cost_point_11", + "cost_point_12", + "output_point_0", + "output_point_1", + "output_point_2", + "output_point_3", + "output_point_4", + "output_point_5", + "output_point_6", + "output_point_7", + "output_point_8", + "output_point_9", + "output_point_10", + "output_point_11", + "output_point_12", + "base_mva", + "variable_cost", + "fixed_cost", + "startup_cost", + "shutdown_cost", + "curtailment_cost", + "power_factor", + "unit_type", + "category", + "cold_start_time", + "warm_start_time", + "hot_start_time", + "startup_ramp", + "shutdown_ramp", + "status_at_start", + "time_at_status", + "cold_start_cost", + "warm_start_cost", + "hot_start_cost", + "must_run", + "pump_load", + "pump_active_power_limits_max", + "pump_active_power_limits_min", + "pump_reactive_power_limits_max", + "pump_reactive_power_limits_min", + "pump_min_down_time", + "pump_min_up_time", + "pump_ramp_limits", + "pump_ramp_up", + "pump_ramp_down", + "generator_category" + ] + }, "valid_branch_types": [ "Line", "TapTransformer", diff --git a/src/r2x/exporter/plexos.py b/src/r2x/exporter/plexos.py index 1022ffa4..645b96e5 100644 --- a/src/r2x/exporter/plexos.py +++ b/src/r2x/exporter/plexos.py @@ -899,11 +899,12 @@ def apply_operation_cost(component: dict) -> dict[str, Any]: def _variable_type_parsing(component: dict, cost_dict: dict[str, Any]) -> dict[str, Any]: fuel_curve = cost_dict["variable"] + value_curve_type = cost_dict["value_curve_type"] variable_type = cost_dict["variable_type"] function_data = fuel_curve["value_curve"]["function_data"] match variable_type: case "FuelCurve": - match fuel_curve["value_curve_type"]: + match value_curve_type: case "AverageRateCurve": component["Heat Rate"] = function_data["proportional_term"] case "InputOutputCurve": diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 3cac628d..af15d604 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -1,17 +1,18 @@ """R2X Sienna system exporter.""" # System packages -from functools import partial import json import os +from functools import partial from typing import Any +from urllib.request import urlopen # Third-party packages from loguru import logger # Local imports from r2x.exporter.handler import BaseExporter, get_export_records -from r2x.exporter.utils import apply_pint_deconstruction, apply_property_map +from r2x.exporter.utils import apply_pint_deconstruction, apply_property_map, apply_unnest_key from r2x.models import ( ACBranch, Bus, @@ -23,6 +24,10 @@ ReserveMap, Storage, ) +from r2x.utils import haskey + +PSY_URL = "https://raw.githubusercontent.com/NREL-Sienna/PowerSystems.jl/refs/heads/main/" +TABLE_DATA_SPEC = "src/descriptors/power_system_inputs.json" class SiennaExporter(BaseExporter): @@ -56,6 +61,12 @@ def __init__(self, *args, **kwargs): self.output_data = {} self.property_map = self.config.defaults.get("sienna_property_map", {}) self.unit_map = self.config.defaults.get("sienna_unit_map", {}) + self.output_fields = self.config.defaults["table_data"] + + def _get_table_data_fields(self) -> dict[str, Any]: + request = urlopen(PSY_URL + TABLE_DATA_SPEC) + descriptor = json.load(request) + return descriptor def run(self, *args, path=None, **kwargs) -> "SiennaExporter": """Run sienna exporter workflow. @@ -210,38 +221,23 @@ def process_gen_data(self, fname="gen.csv"): fname : str Name of the file to be created """ - output_fields = [ - "name", - "available", - "prime_mover_type", - "bus_id", - "fuel", - "base_mva", - "rating", - "unit_type", - "active_power", - "min_rated_capacity", - "min_down_time", - "min_up_time", - "mean_time_to_repair", - "forced_outage_rate", - "planned_outage_rate", - "ramp_up", - "ramp_down", - "category", - "must_run", - "pump_load", - "vom_price", - "operation_cost", - ] - key_mapping = {"bus": "bus_id"} - self.system.export_component_to_csv( - Generator, + + records = [ + component.model_dump(exclude_none=True, mode="python", serialize_as_any=True) + for component in self.system.get_components(Generator) + ] + export_records = get_export_records( + records, + partial(apply_operation_table_data), + partial(apply_property_map, property_map=self.property_map | key_mapping), + partial(apply_pint_deconstruction, unit_map=self.unit_map), + partial(apply_unnest_key, key_map={"bus_id": "number"}), + ) + self.system._export_dict_to_csv( + export_records, fpath=self.output_folder / fname, - fields=output_fields, - key_mapping=key_mapping, - unnest_key="number", + fields=self.output_fields["generator"], restval="NA", ) logger.info(f"File {fname} created.") @@ -432,3 +428,127 @@ def export_data(self) -> None: # First export all time series objects self.export_data_files() logger.info("Saving time series data.") + + +def apply_operation_table_data( + component: dict[str, Any], +) -> dict[str, Any]: + """Process and apply operation cost data for `PSY.power_system_table_data.jl`. + + This function extracts operation cost data from a component dictionary and adds + various cost-related fields to it. It handles different types of cost data, + including variable costs, fuel costs, heat rates, and cost/fuel curves. + + Parameters + ---------- + component : dict[str, Any] + A dictionary containing component data, potentially including operation cost information. + + Returns + ------- + dict[str, Any] + The input component dictionary, potentially modified with additional operation cost fields. + + Notes + ----- + This function modifies the input dictionary in-place and returns it. + + The function processes the following types of operation costs: + - Variable costs (vom_cost) + - Fuel costs + - Heat rate coefficients (a0, a1, a2) + - Cost curves + - Fuel curves + + Raises + ------ + AssertionError + If the fuel_cost is None when present in the variable cost data. + NotImplementedError + If an unsupported variable curve type is encountered. + + Examples + -------- + >>> component = { + ... "operation_cost": { + ... "variable": { + ... "vom_cost": {"function_data": {"proportional_term": 10}}, + ... "fuel_cost": 0.05, + ... "value_curve": { + ... "function_data": { + ... "constant_term": 100, + ... "proportional_term": 20, + ... "quadratic_term": 0.5, + ... "x_coords": [0, 50, 100], + ... "y_coords": [0, 1000, 2500], + ... } + ... }, + ... }, + ... "variable_type": "CostCurve", + ... } + ... } + >>> updated_component = apply_operation_table_data(component) + >>> updated_component["variable_cost"] + 10 + >>> updated_component["fuel_price"] + 50.0 + >>> updated_component["heat_rate_a0"] + 100 + >>> updated_component["output_point_1"] + 50 + >>> updated_component["cost_point_1"] + 1000 + """ + if not component.get("operation_cost", False): + return component + + operation_cost = component["operation_cost"] + + if not (variable := operation_cost.get("variable")): + return component + + if haskey(variable, ["vom_cost", "function_data"]): + component["variable_cost"] = variable["vom_cost"]["function_data"]["proportional_term"] + + if "fuel_cost" in variable.keys(): + assert variable["fuel_cost"] is not None + # Note: We multiply the fuel price by 1000 to offset the division + # done by Sienna when it parses .csv files + component["fuel_price"] = variable["fuel_cost"] * 1000 + if haskey(variable, ["value_curve", "function_data"]): + function_data = variable["value_curve"]["function_data"] + if "constant_term" in function_data.keys(): + component["heat_rate_a0"] = function_data["constant_term"] + if "proportional_term" in function_data.keys(): + component["heat_rate_a1"] = function_data["proportional_term"] + if "quadratic_term" in function_data.keys(): + component["heat_rate_a2"] = function_data["quadratic_term"] + if "x_coords" in function_data.keys(): + component = _variable_type_parsing(component, operation_cost) + return component + + +def _variable_type_parsing(component: dict, cost_dict: dict[str, Any]) -> dict[str, Any]: + variable_curve = cost_dict["variable"] + function_data = variable_curve["value_curve"]["function_data"] + x_y_coords = dict(zip(function_data["x_coords"], function_data["y_coords"])) + match cost_dict["variable_type"]: + case "CostCurve": + for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): + output_point_col = f"output_point_{i}" + component[output_point_col] = x_coord + + cost_point_col = f"cost_point_{i}" + component[cost_point_col] = y_coord + + case "FuelCurve": + for i, (x_coord, y_coord) in enumerate(x_y_coords.items()): + output_point_col = f"output_point_{i}" + component[output_point_col] = x_coord + + heat_rate_col = "heat_rate_avg_0" if i == 0 else f"heat_rate_incr_{i}" + component[heat_rate_col] = y_coord + case _: + msg = f"Type {cost_dict['variable_type']} variable curve not supported" + raise NotImplementedError(msg) + return component diff --git a/src/r2x/exporter/utils.py b/src/r2x/exporter/utils.py index 30f22be8..701b3d4d 100644 --- a/src/r2x/exporter/utils.py +++ b/src/r2x/exporter/utils.py @@ -167,6 +167,53 @@ def apply_valid_properties( return {key: value for key, value in component.items() if key in valid_properties} +def apply_unnest_key(component: dict[str, Any], key_map: dict[str, Any]) -> dict[str, Any]: + """Unnest specific nested dictionary values based on a provided key map. + + This function processes a dictionary, potentially containing nested dictionaries, + and unnests specific values based on a provided key map. For each key in the input + dictionary that has a corresponding entry in the key map, if the value is a dictionary, + it extracts the value using the mapped key. + + Parameters + ---------- + component : dict[str, Any] + The input dictionary to process. + key_map : dict[str, Any] + A dictionary mapping keys in the input dictionary to keys in nested dictionaries. + + Returns + ------- + dict[str, Any] + A new dictionary with unnested values based on the key map. + + Examples + -------- + >>> component = { + ... "name": "Example", + ... "config": {"type": "A", "value": 10}, + ... "data": {"content": "Some data"}, + ... } + >>> key_map = {"config": "type", "data": "content"} + >>> apply_unnest_key(component, key_map) + {'name': 'Example', 'config': 'A', 'data': 'Some data'} + + Notes + ----- + - If a key in the input dictionary is not in the key map, its value remains unchanged. + - If a key is in the key map but the corresponding value in the input dictionary + is not a dictionary, the value remains unchanged. + - If a key is in the key map and the corresponding value is a dictionary, but the + mapped key is not in this nested dictionary, the result for this key will be None. + """ + if not key_map: + return component + return { + key: value if not isinstance(value, dict) else value.get(key_map.get(key), value) + for key, value in component.items() + } + + def get_property_magnitude(property_value, to_unit: str | None = None) -> Any: """Return magnitude with the given units for a pint Quantity. diff --git a/tests/test_exporter_utils.py b/tests/test_exporter_utils.py index 1c4cd3e5..0263ab0a 100644 --- a/tests/test_exporter_utils.py +++ b/tests/test_exporter_utils.py @@ -2,6 +2,7 @@ from pint import Quantity from r2x.exporter.utils import ( apply_property_map, + apply_unnest_key, apply_valid_properties, apply_pint_deconstruction, get_property_magnitude, @@ -74,3 +75,56 @@ def test_get_property_magnitude(): assert get_property_magnitude(q3) == 200 # No conversion for a non-Quantity assert get_property_magnitude(q1) == 100 # Magnitude of Quantity without conversion + + +def test_apply_unnest_key_basic_functionality(): + # Test basic functionality + component = {"name": "Example", "config": {"type": "A", "value": 10}, "data": {"content": "Some data"}} + key_map = {"config": "type", "data": "content"} + result = apply_unnest_key(component, key_map) + + assert result == {"name": "Example", "config": "A", "data": "Some data"} + + # Test no change when key is not in key_map + component = {"name": "Example", "value": 42} + key_map = {"config": "type"} + result = apply_unnest_key(component, key_map) + assert result == component + + # Test nested dictionary with no corresponding key in key_map + component = {"name": "Example", "config": {"type": "A", "value": 10}} + key_map = {"name": "type"} + result = apply_unnest_key(component, key_map) + assert result == component + + +def test_apply_unnest_key_edge_cases(): + # Test missing nested key + component = {"config": {"value": 10}, "data": {"content": "Some data"}} + key_map = {"config": "type", "data": "content"} + result = apply_unnest_key(component, key_map) + assert result == {"config": {"value": 10}, "data": "Some data"} + + # Test empty input + assert apply_unnest_key({}, {"config": "type"}) == {} + + # Test empty key_map + component = {"name": "Example", "config": {"type": "A", "value": 10}} + assert apply_unnest_key(component, {}) == component + + +# Parameterized tests +@pytest.mark.parametrize( + "component,key_map,expected", + [ + ({"a": {"x": 1, "y": 2}, "b": {"z": 3}}, {"a": "x", "b": "z"}, {"a": 1, "b": 3}), + ( + {"a": 1, "b": {"x": 2, "y": 3}, "c": "test"}, + {"b": "y", "c": "nonexistent"}, + {"a": 1, "b": 3, "c": "test"}, + ), + ({"a": {"x": {"nested": "value"}}, "b": 2}, {"a": "x", "b": "y"}, {"a": {"nested": "value"}, "b": 2}), + ], +) +def parameterized_test(component, key_map, expected): + assert apply_unnest_key(component, key_map) == expected diff --git a/tests/test_plexos_exporter.py b/tests/test_plexos_exporter.py index 5af31f15..e3090b3e 100644 --- a/tests/test_plexos_exporter.py +++ b/tests/test_plexos_exporter.py @@ -52,3 +52,7 @@ def test_plexos_exporter_run(plexos_exporter, default_scenario, tmp_folder): # Check that time series was created correctly ts_directory = tmp_folder / exporter.ts_directory assert any(ts_directory.iterdir()) + + +@pytest.mark.plexos +def test_plexos_operational_cost(reeds_system, plexos_exporter): ... diff --git a/tests/test_sienna_exporter.py b/tests/test_sienna_exporter.py index bd8c323d..72b2015b 100644 --- a/tests/test_sienna_exporter.py +++ b/tests/test_sienna_exporter.py @@ -1,6 +1,6 @@ import pytest from r2x.config import Scenario -from r2x.exporter.sienna import SiennaExporter +from r2x.exporter.sienna import SiennaExporter, apply_operation_table_data from .models import ieee5bus @@ -51,3 +51,106 @@ def test_sienna_exporter_run(sienna_exporter, tmp_folder): # Check that time series was created correctly ts_directory = tmp_folder / exporter.ts_directory assert any(ts_directory.iterdir()) + + +@pytest.fixture +def sample_component(): + return { + "operation_cost": { + "variable": { + "vom_cost": {"function_data": {"proportional_term": 10}}, + "fuel_cost": 0.05, + "value_curve": { + "function_data": { + "constant_term": 100, + "proportional_term": 20, + "quadratic_term": 0.5, + "x_coords": [0, 50, 100], + "y_coords": [0, 1000, 2500], + } + }, + }, + "variable_type": "CostCurve", + } + } + + +def test_apply_operation_table_data_basic(sample_component): + updated_component = apply_operation_table_data(sample_component) + + assert "variable_cost" in updated_component + assert "fuel_price" in updated_component + assert updated_component["variable_cost"] == 10 + assert updated_component["fuel_price"] == 50 # 0.05 * 1000 + + +def test_apply_operation_table_data_heat_rate(sample_component): + updated_component = apply_operation_table_data(sample_component) + + assert "heat_rate_a0" in updated_component + assert "heat_rate_a1" in updated_component + assert "heat_rate_a2" in updated_component + assert updated_component["heat_rate_a0"] == 100 + assert updated_component["heat_rate_a1"] == 20 + assert updated_component["heat_rate_a2"] == 0.5 + + +def test_apply_operation_table_data_cost_curve(sample_component): + updated_component = apply_operation_table_data(sample_component) + + assert "output_point_0" in updated_component + assert "cost_point_0" in updated_component + assert updated_component["output_point_0"] == 0 + assert updated_component["cost_point_0"] == 0 + assert updated_component["output_point_1"] == 50 + assert updated_component["cost_point_1"] == 1000 + + +def test_apply_operation_table_data_fuel_curve(): + fuel_curve_component = { + "operation_cost": { + "variable": { + "value_curve": {"function_data": {"x_coords": [0, 50, 100], "y_coords": [0, 10, 25]}} + }, + "variable_type": "FuelCurve", + } + } + updated_component = apply_operation_table_data(fuel_curve_component) + + assert "output_point_0" in updated_component + assert "heat_rate_avg_0" in updated_component + assert updated_component["output_point_0"] == 0 + assert updated_component["heat_rate_avg_0"] == 0 + assert updated_component["output_point_1"] == 50 + assert updated_component["heat_rate_incr_1"] == 10 + + +def test_apply_operation_table_data_no_operation_cost(): + component = {"id": "test_component"} + updated_component = apply_operation_table_data(component) + assert updated_component == component + + +def test_apply_operation_table_data_no_variable(): + component = {"operation_cost": {}} + updated_component = apply_operation_table_data(component) + assert updated_component == component + + +def test_apply_operation_table_data_unsupported_curve(): + component = { + "operation_cost": { + "variable": { + "value_curve": {"function_data": {"x_coords": [0, 50, 100], "y_coords": [0, 1000, 2500]}} + }, + "variable_type": "UnsupportedCurve", + } + } + with pytest.raises(NotImplementedError): + apply_operation_table_data(component) + + +def test_apply_operation_table_data_none_fuel_cost(): + component = {"operation_cost": {"variable": {"fuel_cost": None}}} + with pytest.raises(AssertionError): + apply_operation_table_data(component)