feat: Multiple updates to the Plexos parser (#47)

---------

Co-authored-by: pesap <[email protected]>

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: check-added-large-files
-        args: ["--maxkb=5000"]
+        args: ["--maxkb=10000"]
       - id: check-json
       - id: pretty-format-json
         args: ["--autofix"]

environment.yml

@@ -3,5 +3,5 @@ channels:
   - conda-forge
 
 dependencies:
-  - python>=3.11
+  - python>=3.11,<3.13
   - pip

pyproject.toml

@@ -17,7 +17,7 @@ authors = [
     { name = "Vivienne Liu", email = "[email protected]"},
 ]
 description = "ReEDS to X parser"
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.13"
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
@@ -28,15 +28,16 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-    "infrasys @ git+https://github.com/NREL/infrasys@ps/value_curves",
+    "infrasys~=0.1.1",
     "jsonschema~=4.23",
     "loguru~=0.7.2",
     "pandas~=2.2",
-    "plexosdb>=0.0.4",
+    "plexosdb~=0.0.6",
     "polars~=1.1.0",
     "pyyaml~=6.0.1",
     "rich~=13.7.1",
     "tables~=3.9.2",
+    "cvxpy~=1.5.3",
 ]
 
 [project.optional-dependencies]

src/r2x/cli_functions.py

@@ -2,9 +2,17 @@
 
 import argparse
 import importlib
-from importlib.resources import files
 from .__version__ import __version__
 
+FILES_WITH_ARGS = [
+    "r2x.plugins.pcm_defaults",
+    "r2x.plugins.break_gens",
+    "r2x.parser.plexos",
+    "r2x.parser.reeds",
+    "r2x.exporter.plexos",
+    "r2x.exporter.sienna",
+]
+
 
 class Flags(argparse.Action):
     """Class to enable feature flags on the code and CLI.
@@ -32,7 +40,7 @@ def __call__(  # type: ignore
 
 def get_additional_arguments(
     parser: argparse.ArgumentParser,
-    folder: str = "r2x.plugins",
+    folders: list[str] = FILES_WITH_ARGS,
 ) -> argparse.ArgumentParser:
     """Add plugin arguments to CLI.
 
@@ -41,16 +49,20 @@ def get_additional_arguments(
         plugin_list: List of plugins.
         folder: Folder that contains the plugin.
     """
-    folders = ["r2x.plugins", "r2x.parser", "r2x.exporter"]
+    for package in folders:
+        package_str = package.split(".")
+        if not len(package_str) == 3:
+            msg = (
+                "Format of `FILES_WITH_ARGS` should be `r2x.package.script` but received {package_name}."
+                "Modify `FILES_WITH_ARGS` to match the format."
+            )
+            raise NotImplementedError(msg)
 
-    for folder in folders:
-        folder_files = [folder for folder in files(folder).rglob("*.py") if "__" not in folder.name]  # type: ignore
-        package_name = folder.split(".")[1]
-        for package in folder_files:
-            package_script = importlib.import_module(f".{package.stem}", folder)
-            if hasattr(package_script, "cli_arguments"):
-                script_cli_group = parser.add_argument_group(f"{package_name.upper()}: {package.stem}")
-                package_script.cli_arguments(script_cli_group)
+        _, package_name, script_name = package_str
+        package_script = importlib.import_module(f"{package}")
+        if hasattr(package_script, "cli_arguments"):
+            script_cli_group = parser.add_argument_group(f"{package_name.upper()}: {script_name}")
+            package_script.cli_arguments(script_cli_group)
 
     return parser
 

src/r2x/defaults/config.json

@@ -140,6 +140,10 @@
         "fuel": "GAS",
         "type": null
       },
+      {
+        "fuel": "OT",
+        "type": null
+      },
       {
         "fuel": "OTHER_GAS",
         "type": null

src/r2x/defaults/plexos_input.json

@@ -1,7 +1,9 @@
 {
+  "plexos_category_map": {},
   "plexos_device_map": {},
   "plexos_fuel_map": {},
   "plexos_input_property_map": {
+    "Capacity": "storage_capacity",
     "Charge Efficiency": "charge_efficiency",
     "Commit": "must_run",
     "Discharge Efficiency": "discharge_efficiency",
@@ -17,8 +19,9 @@
     "Load Risk": "load_risk",
     "Loss Incr": "losses",
     "Maintenance Rate": "planned_outage_rate",
-    "Max Capacity": "rating",
+    "Max Capacity": "base_power",
     "Max Flow": "max_power_flow",
+    "Max Power": "base_power",
     "Max Ramp Down": "ramp_down",
     "Max Ramp Up": "ramp_up",
     "Max SoC": "max_storage_capacity",
@@ -34,8 +37,8 @@
     "Pump Efficiency": "pump_efficiency",
     "Pump Load": "pump_load",
     "Rating": "max_active_power",
-    "Reactance": "reactance",
-    "Resistance": "resistance",
+    "Reactance": "x",
+    "Resistance": "r",
     "Start Cost": "startup_cost",
     "Timeframe": "time_frame",
     "Units": "available",

src/r2x/defaults/plexos_output.json

@@ -107,20 +107,19 @@
     "planned_outage_rate": "Maintenance Rate",
     "pump_efficiency": "Pump Efficiency",
     "pump_load": "Pump Load",
+    "r": "Resistance",
     "ramp_down": "Max Ramp Down",
     "ramp_up": "Max Ramp Up",
     "rate": "Production Rate",
-    "rating": "Rating",
     "rating_down": "Min Flow",
     "rating_up": "Max Flow",
-    "reactance": "Reactance",
-    "resistance": "Resistance",
     "startup_cost": "Start Cost",
     "storage_capacity": "Max Volume",
     "time_frame": "Timeframe",
     "voll": "VoLL",
     "vom_price": "VO&M Charge",
-    "vors": "VoRS"
+    "vors": "VoRS",
+    "x": "Reactance"
   },
   "plexos_reports": "r2x/defaults/plexos_reports.json",
   "pv_profile_file_prefix": "PV_profiles",

src/r2x/enums.py

@@ -30,6 +30,7 @@ class ACBusTypes(StrEnum):
     PV = "PV"
     PQ = "PQ"
     REF = "REF"
+    SLACK = "SLACK"
 
 
 class PrimeMoversType(StrEnum):

src/r2x/exporter/handler.py

@@ -99,7 +99,7 @@ def read_file(self, fpath: Path | str, filter_func: list[Callable] | None = None
                 data = func(data, **kwargs)
         return data
 
-    def export_data_files(self, time_series_folder: str = "Data") -> None:
+    def export_data_files(self, year: int, time_series_folder: str = "Data") -> None:
         """Export all time series objects attached to components.
 
         This method assumes that `self.config.weather_year and `self.output_folder` exist.
@@ -109,41 +109,62 @@ def export_data_files(self, time_series_folder: str = "Data") -> None:
         time_series_folder: str
             Folder name to save time series data
         """
+        assert year is not None
         config_dict = self.config.__dict__
         for component in self.system.iter_all_components():
             if self.system.has_time_series(component):
                 for ts_metadata in self.system.time_series.list_time_series_metadata(component):
                     ts_component_name = f"{component.__class__.__name__}_{ts_metadata.variable_name}"
-                    self.time_series_objects[ts_component_name].append(
-                        self.system.get_time_series(component, variable_name=ts_metadata.variable_name)
-                    )
+                    try:
+                        self.time_series_objects[ts_component_name].append(
+                            self.system.get_time_series(component, variable_name=ts_metadata.variable_name)
+                        )
+                    except:  # noqa: E722
+                        continue
                     self.time_series_name_by_type[ts_component_name].append(component.name)
 
-        assert self.config.weather_year is not None
-        date_time_column = pd.date_range(
-            start=f"1/1/{self.config.weather_year}",
-            end=f"1/1/{self.config.weather_year + 1}",
-            freq="1h",
-            inclusive="left",
-        )
-        date_time_column = np.datetime_as_string(date_time_column, unit="m")
-        # Remove leap day to match ReEDS convention
-        # date_time_column = date_time_column[~((date_time_column.month == 2) & (date_time_column.day == 29))]
-        if self.input_model == "reeds-US":
-            date_time_column = date_time_column[:-24]
-
+        component_lengths = {
+            component_type: {ts.length}
+            for component_type, time_series in self.time_series_objects.items()
+            for ts in time_series
+        }
+
+        inconsistent_lengths = [
+            (component_type, length_set)
+            for component_type, length_set in component_lengths.items()
+            if len(length_set) != 1
+        ]
+        if inconsistent_lengths:
+            raise ValueError(f"Multiple lengths found for components time series: {inconsistent_lengths}")
+
+        datetime_arrays = {
+            component_type: (
+                np.datetime_as_string(
+                    pd.date_range(
+                        start=f"1/1/{year}",
+                        periods=ts.length,
+                        freq=f"{int(ts.resolution.total_seconds() / 60)}min",  # Convert resolution to minutes
+                    ),
+                    unit="m",
+                ),
+                time_series,
+            )
+            for component_type, time_series in self.time_series_objects.items()
+            for ts in time_series
+        }
         csv_fpath = self.output_folder / time_series_folder
 
         # Use string substitution to dynamically change the output csv fnames
         csv_fname = config_dict.get("time_series_fname", "${component_type}_${name}_${weather_year}.csv")
+        logger.trace("Using {} as time_series name", csv_fname)
         string_template = string.Template(csv_fname)
 
-        for component_type, time_series in self.time_series_objects.items():
+        for component_type, (datetime_array, time_series) in datetime_arrays.items():
             time_series_arrays = list(map(lambda x: x.data.to_numpy(), time_series))
 
             config_dict["component_type"] = component_type
             csv_fname = string_template.safe_substitute(config_dict)
-            csv_table = np.column_stack([date_time_column, *time_series_arrays])
+            csv_table = np.column_stack([datetime_array, *time_series_arrays])
             header = '"DateTime",' + ",".join(
                 [f'"{name}"' for name in self.time_series_name_by_type[component_type]]
             )

src/r2x/exporter/plexos.py

@@ -41,6 +41,7 @@
     Transformer2W,
     TransmissionInterface,
 )
+from r2x.models.branch import Line
 from r2x.units import get_magnitude
 from r2x.utils import custom_attrgetter, get_enum_from_string, read_json
 
@@ -76,6 +77,16 @@ def __init__(
         self.default_units = self.config.defaults["default_units"]
         self.reserve_types = self.config.defaults["reserve_types"]
 
+        if not isinstance(self.config.solve_year, int):
+            msg = "Multiple solve years are not supported yet."
+            raise NotImplementedError(msg)
+        self.year: int = self.config.solve_year
+
+        if not isinstance(self.config.weather_year, int):
+            msg = "Multiple solve years are not supported yet."
+            raise NotImplementedError(msg)
+        self.weather_year: int = self.config.weather_year
+
         if not xml_fname:
             if not (xml_fname := getattr(self.config, "master_file", None)):
                 xml_fname = files("r2x.defaults").joinpath(DEFAULT_XML_TEMPLATE)  # type: ignore
@@ -86,7 +97,7 @@ def run(self, *args, new_database: bool = True, **kwargs) -> "PlexosExporter":
         """Run the exporter."""
         logger.info("Starting {}", self.__class__.__name__)
 
-        self.export_data_files()
+        self.export_data_files(year=self.weather_year)
 
         # If starting w/o a reference file we add our custom models and objects
         if new_database:
@@ -201,6 +212,8 @@ def insert_component_properties(
         match component_type.__name__:
             case "GenericBattery":
                 custom_map = {"active_power": "Max Power", "storage_capacity": "Capacity"}
+            case "Line":
+                custom_map = {"rating": "Max Flow"}
             case _:
                 custom_map = {}
         property_map = self.property_map | custom_map
@@ -237,7 +250,9 @@ def add_component_category(
         class_id = self._db_mgr.get_class_id(class_enum)
         categories = [
             (class_id, rank, category or "")
-            for rank, category in enumerate(sorted(component_categories), start=existing_rank + 1)
+            for rank, category in enumerate(
+                sorted(component_categories, key=lambda x: (x is None, x)), start=existing_rank + 1
+            )
         ]
 
         # Maybe replace `t_category` with right schema.
@@ -405,6 +420,8 @@ def add_lines(self) -> None:
         # NOTE: The default line on Plexos is a `MonitoredLine` without category. If we need to add a category
         # in the future, we will uncomment the line below with the pertinent category name.
         # self.add_component_category(MonitoredLine, class_enum=ClassEnum.Line)
+        self.bulk_insert_objects(Line, class_enum=ClassEnum.Line, collection_enum=CollectionEnum.Lines)
+        self.insert_component_properties(Line, parent_class=ClassEnum.System, collection=CollectionEnum.Lines)
         self.bulk_insert_objects(
             MonitoredLine, class_enum=ClassEnum.Line, collection_enum=CollectionEnum.Lines
         )
@@ -416,7 +433,7 @@ def add_lines(self) -> None:
         collection_properties = self._db_mgr.get_valid_properties(
             collection=CollectionEnum.Lines, parent_class=ClassEnum.System, child_class=ClassEnum.Line
         )
-        for line in self.system.get_components(MonitoredLine, filter_func=lambda x: getattr(x, "ext", False)):
+        for line in self.system.get_components(MonitoredLine, Line):
             properties = get_export_properties(
                 line.ext,
                 partial(apply_property_map, property_map=self.property_map),