From fe667d7447203bc156ef28d55712079522340c00 Mon Sep 17 00:00:00 2001 From: pesap Date: Tue, 22 Oct 2024 15:01:58 -0600 Subject: [PATCH] test: Adding further testing --- src/r2x/exporter/handler.py | 46 ++++++--- src/r2x/exporter/sienna.py | 14 +-- src/r2x/models/__init__.py | 2 +- src/r2x/models/generators.py | 2 +- src/r2x/parser/parser_helpers.py | 157 ++++++++++++++++++++++++++----- tests/data/pjm_2area_data.json | 4 +- tests/models/pjm.py | 24 ++++- tests/test_models.py | 12 +++ tests/test_operational_cost.py | 57 +++++++++++ tests/test_parser_handler.py | 16 ++++ tests/test_parser_helper.py | 100 ++++++++++++++++++++ tests/test_sienna_exporter.py | 21 ++++- 12 files changed, 400 insertions(+), 55 deletions(-) create mode 100644 tests/test_operational_cost.py create mode 100644 tests/test_parser_handler.py create mode 100644 tests/test_parser_helper.py diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index b5d1c38d..895bf5de 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -123,30 +123,48 @@ def export_data_files(self, year: int, time_series_folder: str = "Data") -> None continue self.time_series_name_by_type[ts_component_name].append(component.name) - date_time_column = pd.date_range( - start=f"1/1/{year}", - end=f"1/1/{year + 1}", - freq="1h", - inclusive="left", - ) - date_time_column = np.datetime_as_string(date_time_column, unit="m") - # Remove leap day to match ReEDS convention - # date_time_column = date_time_column[~((date_time_column.month == 2) & (date_time_column.day == 29))] - if self.input_model == "reeds-US": - date_time_column = date_time_column[:-24] - + component_lengths = { + component_type: {ts.length} + for component_type, time_series in self.time_series_objects.items() + for ts in time_series + } + + inconsistent_lengths = [ + (component_type, length_set) + for component_type, length_set in component_lengths.items() + if len(length_set) != 1 + ] + if inconsistent_lengths: + raise ValueError(f"Multiple lengths found for components time series: {inconsistent_lengths}") + + datetime_arrays = { + component_type: ( + np.datetime_as_string( + pd.date_range( + start=f"1/1/{year}", + periods=ts.length, + freq=f"{int(ts.resolution.total_seconds() / 60)}min", # Convert resolution to minutes + ), + unit="m", + ), + time_series, + ) + for component_type, time_series in self.time_series_objects.items() + for ts in time_series + } csv_fpath = self.output_folder / time_series_folder # Use string substitution to dynamically change the output csv fnames csv_fname = config_dict.get("time_series_fname", "${component_type}_${name}_${weather_year}.csv") + logger.trace("Using {} as time_series name", csv_fname) string_template = string.Template(csv_fname) - for component_type, time_series in self.time_series_objects.items(): + for component_type, (datetime_array, time_series) in datetime_arrays.items(): time_series_arrays = list(map(lambda x: x.data.to_numpy(), time_series)) config_dict["component_type"] = component_type csv_fname = string_template.safe_substitute(config_dict) - csv_table = np.column_stack([date_time_column, *time_series_arrays]) + csv_table = np.column_stack([datetime_array, *time_series_arrays]) header = '"DateTime",' + ",".join( [f'"{name}"' for name in self.time_series_name_by_type[component_type]] ) diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index e71a3f0b..1da0d619 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -35,6 +35,13 @@ TABLE_DATA_SPEC = "src/descriptors/power_system_inputs.json" +def get_psy_fields() -> dict[str, Any]: + """Get PSY JSON schema.""" + request = urlopen(PSY_URL + TABLE_DATA_SPEC) + descriptor = json.load(request) + return descriptor + + class SiennaExporter(BaseExporter): """Sienna exporter class. @@ -73,11 +80,6 @@ def __init__(self, *args, **kwargs): raise NotImplementedError(msg) self.year: int = self.config.solve_year - def _get_table_data_fields(self) -> dict[str, Any]: - request = urlopen(PSY_URL + TABLE_DATA_SPEC) - descriptor = json.load(request) - return descriptor - def run(self, *args, path=None, **kwargs) -> "SiennaExporter": """Run sienna exporter workflow. @@ -386,7 +388,7 @@ def process_storage_data(self, fname="storage.csv") -> None: hydro_pump = list(self.system.to_records(HydroPumpedStorage)) storage_list = generic_storage + hydro_pump - if storage_list is None: + if not storage_list: logger.warning("No storage devices found") return diff --git a/src/r2x/models/__init__.py b/src/r2x/models/__init__.py index e190f494..dcf96c40 100644 --- a/src/r2x/models/__init__.py +++ b/src/r2x/models/__init__.py @@ -1,7 +1,7 @@ # Models # ruff: noqa from .branch import Branch, ACBranch, DCBranch, MonitoredLine, Transformer2W -from .core import ReserveMap, TransmissionInterfaceMap +from .core import ReserveMap, TransmissionInterfaceMap, MinMax from .generators import ( Generator, ThermalGen, diff --git a/src/r2x/models/generators.py b/src/r2x/models/generators.py index 543112bf..10c99fd9 100644 --- a/src/r2x/models/generators.py +++ b/src/r2x/models/generators.py @@ -129,7 +129,7 @@ class Generator(Device): ] = None @field_serializer("active_power_limits") - def serialize_address(self, min_max: MinMax) -> dict[str, Any]: + def serialize_active_power_limits(self, min_max: MinMax) -> dict[str, Any]: if min_max is not None: return {"min": min_max.min, "max": min_max.max} diff --git a/src/r2x/parser/parser_helpers.py b/src/r2x/parser/parser_helpers.py index fd750789..58e38792 100644 --- a/src/r2x/parser/parser_helpers.py +++ b/src/r2x/parser/parser_helpers.py @@ -1,6 +1,7 @@ """Set of helper functions for parsers.""" # ruff: noqa +from typing import Any import polars as pl import numpy as np import cvxpy as cp @@ -8,19 +9,68 @@ from infrasys.function_data import QuadraticFunctionData, PiecewiseLinearData, XYCoords -def field_filter(property_fields, eligible_fields): +def field_filter( + property_fields: dict[str, Any], eligible_fields: set[str] +) -> tuple[dict[str, Any], dict[str, Any]]: + """Filters a dictionary of property fields into valid and extra fields based on eligibility. + + Parameters + ---------- + property_fields : dict + Dictionary of property fields where keys are field names and values are field values. + eligible_fields : set + Set of field names that are considered valid (eligible). + + Returns + ------- + tuple of dict + A tuple of two dictionaries: + - valid : dict + Contains fields that are both in `property_fields` and `eligible_fields`, and are not `None`. + - extra : dict + Contains fields that are in `property_fields` but not in `eligible_fields`, and are not `None`. + + Examples + -------- + >>> property_fields = {"field1": 10, "field2": None, "field3": "hello"} + >>> eligible_fields = {"field1", "field2"} + >>> valid, extra = field_filter(property_fields, eligible_fields) + >>> valid + {'field1': 10} + >>> extra + {'field3': 'hello'} + """ valid = {k: v for k, v in property_fields.items() if k in eligible_fields if v is not None} extra = {k: v for k, v in property_fields.items() if k not in eligible_fields if v is not None} return valid, extra -def prepare_ext_field(valid_fields, extra_fields): - """Cleanses the extra fields by removing any timeseries data.""" +def prepare_ext_field(valid_fields: dict[str, Any], extra_fields: dict[str, Any]) -> dict[str, Any]: + """Clean the extra fields by removing any time series data and adds the cleaned extra fields to `valid_fields`. + + Parameters + ---------- + valid_fields : dict + Dictionary containing valid fields. + extra_fields : dict + Dictionary containing extra fields that may include data types not needed. + + Returns + ------- + dict + Updated valid_fields with cleansed extra fields under the "ext" key. + + Examples + -------- + >>> valid_fields = {"field1": 10, "field2": "hello"} + >>> extra_fields = {"field3": [1, 2, 3], "field4": 42} + >>> result = prepare_ext_field(valid_fields, extra_fields) + >>> result + {'field1': 10, 'field2': 'hello', 'ext': {'field4': 42}} + """ if extra_fields: - # Implement any filtering of ext_data here - # logger.debug("Extra fields: {}", extra_fields) - # remove any non eligible datatypes from extra fields + # Filter to only include eligible data types eligible_datatypes = [str, int, float, bool] extra_fields = {k: v for k, v in extra_fields.items() if type(v) in eligible_datatypes} valid_fields["ext"] = extra_fields @@ -30,7 +80,23 @@ def prepare_ext_field(valid_fields, extra_fields): def handle_leap_year_adjustment(data_file: pl.DataFrame) -> pl.DataFrame: - """Duplicate feb 28th to feb 29th for leap years.""" + """Duplicate February 28th to February 29th for leap years. + + Parameters + ---------- + data_file : pl.DataFrame + DataFrame containing timeseries data. + + Returns + ------- + pl.DataFrame + DataFrame adjusted for leap years. + + Examples + -------- + >>> df = pl.DataFrame({"date": ["2020-02-28"], "value": [1]}) + >>> handle_leap_year_adjustment(df) + """ feb_28 = data_file.slice(1392, 24) before_feb_29 = data_file.slice(0, 1416) after_feb_29 = data_file.slice(1416, len(data_file) - 1440) @@ -38,7 +104,26 @@ def handle_leap_year_adjustment(data_file: pl.DataFrame) -> pl.DataFrame: def fill_missing_timestamps(data_file: pl.DataFrame, hourly_time_index: pl.DataFrame) -> pl.DataFrame: - """Add missing timestamps to data and forward fill nulls to complete a year.""" + """Add missing timestamps to data and forward fill nulls to complete a year. + + Parameters + ---------- + data_file : pl.DataFrame + DataFrame containing timeseries data. + hourly_time_index : pl.DataFrame + DataFrame containing the hourly time index for the study year. + + Returns + ------- + pl.DataFrame + DataFrame with missing timestamps filled. + + Examples + -------- + >>> df = pl.DataFrame({"year": [2020], "month": [2], "day": [28], "hour": [0], "value": [1]}) + >>> hourly_index = pl.DataFrame({"datetime": pl.date_range("2020-01-01", "2020-12-31", freq="1H")}) + >>> fill_missing_timestamps(df, hourly_index) + """ if "hour" in data_file.columns: data_file = data_file.with_columns( pl.datetime(pl.col("year"), pl.col("month"), pl.col("day"), pl.col("hour")) @@ -54,27 +139,49 @@ def fill_missing_timestamps(data_file: pl.DataFrame, hourly_time_index: pl.DataF def resample_data_to_hourly(data_file: pl.DataFrame) -> pl.DataFrame: - """Resample data to hourly frequency from 30 minute data.""" - data_file = data_file.with_columns((pl.col("hour") % 48).alias("hour")) - data_file = ( - data_file.with_columns( - ( - pl.datetime( - data_file["year"], - data_file["month"], - data_file["day"], - hour=data_file["hour"] // 2, - minute=(data_file["hour"] % 2) * 30, - ) - ).alias("timestamp") - ) - .sort("timestamp") - .filter(pl.col("timestamp").is_not_null()) + """Resample data to hourly frequency from minute data. + + Parameters + ---------- + data_file : pl.DataFrame + DataFrame containing timeseries data with minute intervals. + + Returns + ------- + pl.DataFrame + DataFrame resampled to hourly frequency. + + Examples + -------- + >>> df = pl.DataFrame( + ... { + ... "year": [2020, 2020, 2020, 2020], + ... "month": [2, 2, 2, 2], + ... "day": [28, 28, 28, 28], + ... "hour": [0, 0, 1, 1], + ... "minute": [0, 30, 0, 30], # Minute-level data + ... "value": [1, 2, 3, 4], + ... } + ... ) + >>> resampled_data = resample_data_to_hourly(df) + >>> resampled_data.shape[0] + # Expecting two rows: one for hour 0 and one for hour 1 + """ + # Create a timestamp from year, month, day, hour, and minute + data_file = data_file.with_columns( + pl.datetime( + data_file["year"], + data_file["month"], + data_file["day"], + hour=data_file["hour"], + minute=data_file["minute"], + ).alias("timestamp") ) + # Group by the hour and aggregate the values return ( data_file.group_by_dynamic("timestamp", every="1h") - .agg([pl.col("value").mean().alias("value")]) + .agg([pl.col("value").mean().alias("value")]) # Average of values for the hour .with_columns( pl.col("timestamp").dt.year().alias("year"), pl.col("timestamp").dt.month().alias("month"), diff --git a/tests/data/pjm_2area_data.json b/tests/data/pjm_2area_data.json index b2acf323..2e610a99 100644 --- a/tests/data/pjm_2area_data.json +++ b/tests/data/pjm_2area_data.json @@ -1675,6 +1675,8 @@ 0.074636836, 0.083141296, 0.077302516, - 0.078747646 + 0.078747646, + 0.210797797, + 0.195670352 ] } diff --git a/tests/models/pjm.py b/tests/models/pjm.py index f609874e..ae6b4fee 100644 --- a/tests/models/pjm.py +++ b/tests/models/pjm.py @@ -1,14 +1,17 @@ """Script that creates simple 2-area pjm systems for testing.""" -from datetime import datetime, timedelta import pathlib +from datetime import datetime, timedelta from infrasys.time_series_models import SingleTimeSeries + from r2x.api import System -from r2x.enums import ACBusTypes, PrimeMoversType +from r2x.enums import ACBusTypes, PrimeMoversType, ReserveDirection, ReserveType from r2x.models.branch import AreaInterchange, Line, MonitoredLine +from r2x.models.core import ReserveMap from r2x.models.generators import RenewableDispatch, ThermalStandard from r2x.models.load import PowerLoad +from r2x.models.services import Reserve from r2x.models.topology import ACBus, Area, LoadZone from r2x.units import ActivePower, Percentage, Time, Voltage, ureg from r2x.utils import read_json @@ -161,4 +164,21 @@ def pjm_2area() -> System: ) system.add_component(load_component) system.add_time_series(ld_ts, load_component) + + # Create reserve + reserve_map = ReserveMap(name="pjm_reserve_map") + reserve = Reserve( + name="SpinUp-pjm", + region=system.get_component(LoadZone, "LoadZone1"), + reserve_type=ReserveType.SPINNING, + vors=0.05, + duration=3600.0, + load_risk=0.5, + time_frame=3600, + direction=ReserveDirection.UP, + ) + reserve_map.mapping[ReserveType.SPINNING.name].append(wind_01.name) + reserve_map.mapping[ReserveType.SPINNING.name].append(solar_pv_01.name) + system.add_components(reserve, reserve_map) + return system diff --git a/tests/test_models.py b/tests/test_models.py index 9193472f..7b005d88 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,5 +1,6 @@ from r2x.enums import PrimeMoversType from r2x.models import Generator, ACBus, Emission, HydroPumpedStorage, ThermalStandard +from r2x.models import MinMax from r2x.units import EmissionRate, ureg @@ -36,3 +37,14 @@ def test_pumped_hydro_generator(): assert isinstance(pumped_storage.bus, ACBus) assert isinstance(pumped_storage.prime_mover_type, PrimeMoversType) assert pumped_storage.prime_mover_type == PrimeMoversType.PS + + +def test_serialize_active_power_limits(): + active_power_limits = MinMax(min=0, max=100) + generator = Generator(name="TestGEN", active_power_limits=active_power_limits) + + output = generator.model_dump() + assert output["active_power_limits"] == {"min": 0, "max": 100} + + output = generator.serialize_active_power_limits(active_power_limits) + assert output == {"min": 0, "max": 100} diff --git a/tests/test_operational_cost.py b/tests/test_operational_cost.py new file mode 100644 index 00000000..842a8f90 --- /dev/null +++ b/tests/test_operational_cost.py @@ -0,0 +1,57 @@ +from infrasys.cost_curves import FuelCurve +from infrasys.value_curves import LinearCurve +from r2x.models.costs import ( + HydroGenerationCost, + OperationalCost, + RenewableGenerationCost, + StorageCost, + ThermalGenerationCost, +) + + +def test_properties(): + cost = OperationalCost() + assert isinstance(cost, OperationalCost) + + +def test_computed_fields(): + variable = FuelCurve(value_curve=LinearCurve(0)) + cost = ThermalGenerationCost(variable=variable) + + assert isinstance(cost, OperationalCost) + assert isinstance(cost, ThermalGenerationCost) + assert cost.variable_type == "FuelCurve" + assert cost.value_curve_type == "InputOutputCurve" + + cost = ThermalGenerationCost() + assert isinstance(cost, OperationalCost) + assert isinstance(cost, ThermalGenerationCost) + assert cost.variable_type is None + assert cost.value_curve_type is None + + +def test_default_fields(): + cost = ThermalGenerationCost() + assert isinstance(cost, OperationalCost) + assert isinstance(cost, ThermalGenerationCost) + assert cost.fixed == 0.0 + assert cost.shut_down == 0.0 + assert cost.start_up == 0.0 + + cost = HydroGenerationCost() + assert isinstance(cost, OperationalCost) + assert isinstance(cost, HydroGenerationCost) + assert cost.fixed == 0.0 + + cost = RenewableGenerationCost() + assert isinstance(cost, OperationalCost) + assert isinstance(cost, RenewableGenerationCost) + assert cost.curtailment_cost is None + + cost = StorageCost() + assert isinstance(cost, OperationalCost) + assert isinstance(cost, StorageCost) + assert cost.energy_surplus_cost == 0.0 + assert cost.fixed == 0.0 + assert cost.shut_down == 0.0 + assert cost.start_up == 0.0 diff --git a/tests/test_parser_handler.py b/tests/test_parser_handler.py new file mode 100644 index 00000000..31ee0636 --- /dev/null +++ b/tests/test_parser_handler.py @@ -0,0 +1,16 @@ +from pathlib import Path +from tempfile import NamedTemporaryFile + +import pytest + +from r2x.parser.handler import file_handler + + +def test_file_handler(): + file_content = "name,value\nTemp,25.5\nLoad,1200\nPressure,101.3" + with NamedTemporaryFile(mode="w+", suffix=".asd", delete=False) as temp_file: + temp_file.write(file_content) + temp_file.seek(0) + + with pytest.raises(NotImplementedError): + _ = file_handler(Path(temp_file.name)) diff --git a/tests/test_parser_helper.py b/tests/test_parser_helper.py new file mode 100644 index 00000000..8ba531de --- /dev/null +++ b/tests/test_parser_helper.py @@ -0,0 +1,100 @@ +import pytest +import polars as pl + +from r2x.parser.parser_helpers import field_filter, prepare_ext_field, resample_data_to_hourly + + +@pytest.mark.parametrize( + "property_fields, eligible_fields, expected_valid, expected_extra", + [ + ( + {"field1": 10, "field2": None, "field3": "hello"}, + {"field1", "field2"}, + {"field1": 10}, + {"field3": "hello"}, + ), + ({"field1": 10, "field3": "hello"}, {"field2"}, {}, {"field1": 10, "field3": "hello"}), + ({}, {"field1", "field2"}, {}, {}), + ({"field1": 10, "field2": 20}, {"field1", "field2"}, {"field1": 10, "field2": 20}, {}), + ({"field1": None, "field2": 20, "field3": None}, {"field1", "field2"}, {"field2": 20}, {}), + ], +) +def test_field_filter(property_fields, eligible_fields, expected_valid, expected_extra): + valid, extra = field_filter(property_fields, eligible_fields) + assert valid == expected_valid + assert extra == expected_extra + + +def test_prepare_ext_field(): + # Test case 1: With extra fields containing eligible and non-eligible types + valid_fields = {"field1": 10, "field2": "hello"} + extra_fields = {"field3": [1, 2, 3], "field4": 42, "field5": None} + result = prepare_ext_field(valid_fields, extra_fields) + assert result == {"field1": 10, "field2": "hello", "ext": {"field4": 42}} + + # Test case 2: No extra fields + valid_fields = {"field1": 10, "field2": "hello"} + extra_fields = {} + result = prepare_ext_field(valid_fields, extra_fields) + assert result == {"field1": 10, "field2": "hello", "ext": {}} + + # Test case 3: Various types including eligible and non-eligible + valid_fields = {"field1": 10, "field2": "hello"} + extra_fields = { + "field3": [1, 2, 3], # Not eligible + "field4": 42, # Eligible + "field5": "world", # Eligible + "field6": None, # Not eligible + "field7": 3.14, # Eligible + } + result = prepare_ext_field(valid_fields, extra_fields) + assert result == { + "field1": 10, + "field2": "hello", + "ext": {"field4": 42, "field5": "world", "field7": 3.14}, + } + + # Test case 4: All non-eligible extra fields + valid_fields = {"field1": 10, "field2": "hello"} + extra_fields = { + "field3": [1, 2, 3], # Not eligible + "field4": {1: "a"}, # Not eligible + "field5": None, # Not eligible + } + result = prepare_ext_field(valid_fields, extra_fields) + assert result == {"field1": 10, "field2": "hello", "ext": {}} + + +def test_resample_data_to_hourly(): + """Test resampling of half-hourly data to hourly data.""" + # Test case 1: Resampling from half-hourly to hourly + input_data_1 = pl.DataFrame( + { + "year": [2020, 2020], + "month": [2, 2], + "day": [28, 28], + "hour": [0, 0], + "minute": [0, 30], + "value": [1, 2], + } + ) + result_1 = resample_data_to_hourly(input_data_1) + assert len(result_1) == 1 # Expecting 1 hourly value + assert result_1["value"].to_list() == [1.5] # Expected average value + + input_data_2 = pl.DataFrame( + { + "year": [2020, 2020, 2020], + "month": [2, 2, 2], + "day": [28, 28, 28], + "hour": [0, 1, 1], + "minute": [0, 0, 30], + "value": [1, None, 3], + } + ) + + result_2 = resample_data_to_hourly(input_data_2) + + # Check the result length and values + assert len(result_2) == 2 # Expecting 2 hourly values + assert result_2["value"].to_list() == [1.0, 3.0] # Expected filled values diff --git a/tests/test_sienna_exporter.py b/tests/test_sienna_exporter.py index 8012b855..ac33021d 100644 --- a/tests/test_sienna_exporter.py +++ b/tests/test_sienna_exporter.py @@ -1,12 +1,13 @@ import pytest + from r2x.config import Scenario -from r2x.exporter.sienna import SiennaExporter, apply_operation_table_data -from .models import ieee5bus +from r2x.exporter.sienna import SiennaExporter, apply_operation_table_data, get_psy_fields +from tests.models.pjm import pjm_2area @pytest.fixture def infrasys_test_system(): - return ieee5bus() + return pjm_2area() @pytest.fixture @@ -39,8 +40,8 @@ def test_sienna_exporter_run(sienna_exporter, tmp_folder): "gen.csv", "bus.csv", "timeseries_pointers.json", - "storage.csv", - # "reserves.csv", # Reserve could be optional + # "storage.csv", # Storage is also optional + "reserves.csv", # Reserve could be optional "dc_branch.csv", "branch.csv", ] @@ -53,6 +54,11 @@ def test_sienna_exporter_run(sienna_exporter, tmp_folder): assert any(ts_directory.iterdir()) +def test_sienna_exporter_empty_storage(caplog, sienna_exporter): + sienna_exporter.process_storage_data() + assert "No storage devices found" in caplog.text + + @pytest.fixture def sample_component(): return { @@ -74,6 +80,11 @@ def sample_component(): } +def test_get_psy_fields(): + fields = get_psy_fields() + assert isinstance(fields, dict) + + def test_apply_operation_table_data_basic(sample_component): updated_component = apply_operation_table_data(sample_component)