diff --git a/notebooks/work_in_progress/sandbox.ipynb b/notebooks/work_in_progress/sandbox.ipynb index 31dd515..d3c6937 100644 --- a/notebooks/work_in_progress/sandbox.ipynb +++ b/notebooks/work_in_progress/sandbox.ipynb @@ -22,7 +22,7 @@ "\n", "import oge.download_data as download_data\n", "import oge.load_data as load_data\n", - "from oge.column_checks import get_dtypes, apply_dtypes\n", + "from oge.column_checks import get_dtypes, apply_dtypes, DATA_COLUMNS\n", "from oge.filepaths import *\n", "import oge.impute_hourly_profiles as impute_hourly_profiles\n", "import oge.data_cleaning as data_cleaning\n", diff --git a/src/oge/consumed.py b/src/oge/consumed.py index ea8cbb2..11e0e42 100644 --- a/src/oge/consumed.py +++ b/src/oge/consumed.py @@ -402,6 +402,8 @@ def _load_rates(self): for f in os.listdir( results_folder(f"{self.prefix}/power_sector_data/hourly/us_units/") ): + # TODO: delete this message, for testing + logger.info(f"Loading {f}") if ".DS_Store" in f: continue this_ba = pd.read_csv( diff --git a/src/oge/helpers.py b/src/oge/helpers.py index f578c40..392a988 100644 --- a/src/oge/helpers.py +++ b/src/oge/helpers.py @@ -440,20 +440,6 @@ def combine_subplant_data( copy=False, ) - # to save memory, delete any rows where all of the data columns are zero - combined_subplant_data = combined_subplant_data[ - combined_subplant_data[DATA_COLUMNS].sum(axis=1) != 0 - ] - - # groupby subplant after combining in case subplant reported multiple places - combined_subplant_data = ( - combined_subplant_data.groupby(KEY_COLUMNS, dropna=False, sort=False)[ - DATA_COLUMNS - ] - .sum(numeric_only=True) - .reset_index() - ) - # re-order the columns combined_subplant_data = combined_subplant_data[ALL_COLUMNS]