From 9cea5a470206b2a9798f9be0f6099f6f789e1a15 Mon Sep 17 00:00:00 2001 From: Christina Gosnell Date: Mon, 2 Dec 2024 17:23:17 -0500 Subject: [PATCH] move release notes docs --- docs/release_notes.rst | 22 ++----------------- test/conftest.py | 6 +++++ test/validate/ferc1_test.py | 44 ++++++++++++++++--------------------- 3 files changed, 27 insertions(+), 45 deletions(-) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 7ed0d2f8b0..ea5857714b 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -22,6 +22,8 @@ Bug Fixes :ref:`core_ferc1__yearly_other_regulatory_liabilities_sched278`. See issue :issue:`3952` and PRs :pr:`3969,3979`. Thanks to :user:`yolandazzz13` for making this fix. +* Added preliminary data validation checks for several FERC 1 tables that were + missing it :pr:`3860`. Major Dependency Updates ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -139,26 +141,6 @@ Bug Fixes * Fixed a bug found in the rolling averages used to impute missing values in ``fuel_cost_per_mmbtu`` and to calculate ``capex_annual_addition_rolling``. Thanks to RMI for identifying this bug! See issue :issue:`3889` and PR :pr:`3892`. -* Added preliminary data validation checks for several FERC 1 tables that were - missing it: - - * out_ferc1__yearly_balance_sheet_assets_sched110 - * out_ferc1__yearly_balance_sheet_liabilities_sched110 - * out_ferc1__yearly_cash_flows_sched120 - * out_ferc1__yearly_depreciation_by_function_sched219 - * out_ferc1__yearly_depreciation_changes_sched219 - * out_ferc1__yearly_depreciation_summary_sched336 - * out_ferc1__yearly_energy_dispositions_sched401 - * out_ferc1__yearly_energy_sources_sched401 - * out_ferc1__yearly_income_statements_sched114 - * out_ferc1__yearly_operating_expenses_sched320 - * out_ferc1__yearly_operating_revenues_sched300 - * out_ferc1__yearly_other_regulatory_liabilities_sched278 - * out_ferc1__yearly_retained_earnings_sched118 - * out_ferc1__yearly_sales_by_rate_schedules_sched304 - * out_ferc1__yearly_small_plants_sched410 - * out_ferc1__yearly_transmission_lines_sched422 - * out_ferc1__yearly_utility_plant_summary_sched200 Major Dependency Updates ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/test/conftest.py b/test/conftest.py index 944d50df88..55994116db 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -109,6 +109,12 @@ def live_databases(request) -> bool: @pytest.fixture(scope="session") def asset_value_loader() -> AssetValueLoader: + """Fixture that initializes an asset value loader. + + Use this as ``asset_value_loader.load_asset_value`` instead + of ``defs.load_asset_value`` to not reinitialize the asset + value loader over and over again. + """ return defs.get_asset_value_loader() diff --git a/test/validate/ferc1_test.py b/test/validate/ferc1_test.py index a5d1e93640..92991ece17 100644 --- a/test/validate/ferc1_test.py +++ b/test/validate/ferc1_test.py @@ -7,6 +7,7 @@ import logging from itertools import chain +import pandas as pd import pytest from pudl import validate as pv @@ -85,41 +86,35 @@ def test_no_null_cols_ferc1(live_dbs, asset_value_loader, cols, asset_key): ) -# TODO 2024-09-20: many of these row counts haven't actually been confirmed to -# be "correct", they're just "what the values were when we added them to the -# test." Those tables are marked as "unverified" below @pytest.mark.parametrize( "asset_key,expected_rows", [ ("_out_ferc1__yearly_plants_utilities", 7_887), ("out_ferc1__yearly_all_plants", 58_520), - ("out_ferc1__yearly_balance_sheet_assets_sched110", 278_789), # unverified - ("out_ferc1__yearly_balance_sheet_liabilities_sched110", 233_383), # unverified - ("out_ferc1__yearly_cash_flows_sched120", 306_837), # unverified - ("out_ferc1__yearly_depreciation_by_function_sched219", 148_352), # unverified - ("out_ferc1__yearly_depreciation_changes_sched219", 263_942), # unverified - ("out_ferc1__yearly_depreciation_summary_sched336", 216_710), # unverified - ("out_ferc1__yearly_energy_dispositions_sched401", 25_954), # unverified - ("out_ferc1__yearly_energy_sources_sched401", 38_315), # unverified + ("out_ferc1__yearly_balance_sheet_assets_sched110", 278_789), + ("out_ferc1__yearly_balance_sheet_liabilities_sched110", 233_383), + ("out_ferc1__yearly_cash_flows_sched120", 306_837), + ("out_ferc1__yearly_depreciation_by_function_sched219", 148_352), + ("out_ferc1__yearly_depreciation_changes_sched219", 263_942), + ("out_ferc1__yearly_depreciation_summary_sched336", 216_710), + ("out_ferc1__yearly_energy_dispositions_sched401", 25_954), + ("out_ferc1__yearly_energy_sources_sched401", 38_315), ("out_ferc1__yearly_hydroelectric_plants_sched406", 7_202), - ("out_ferc1__yearly_income_statements_sched114", 347_394), # unverified - ("out_ferc1__yearly_operating_expenses_sched320", 618_518), # unverified - ("out_ferc1__yearly_operating_revenues_sched300", 77_646), # unverified - ( - "out_ferc1__yearly_other_regulatory_liabilities_sched278", - 53015, - ), # unverified + ("out_ferc1__yearly_income_statements_sched114", 347_394), + ("out_ferc1__yearly_operating_expenses_sched320", 618_518), + ("out_ferc1__yearly_operating_revenues_sched300", 77_646), + ("out_ferc1__yearly_other_regulatory_liabilities_sched278", 53015), ("out_ferc1__yearly_plant_in_service_sched204", 355_918), ("out_ferc1__yearly_pumped_storage_plants_sched408", 580), ("out_ferc1__yearly_purchased_power_and_exchanges_sched326", 211_794), - ("out_ferc1__yearly_retained_earnings_sched118", 105_585), # unverified - ("out_ferc1__yearly_sales_by_rate_schedules_sched304", 304_009), # unverified - ("out_ferc1__yearly_small_plants_sched410", 17763), # unverified + ("out_ferc1__yearly_retained_earnings_sched118", 105_585), + ("out_ferc1__yearly_sales_by_rate_schedules_sched304", 303_909), + ("out_ferc1__yearly_small_plants_sched410", 17763), ("out_ferc1__yearly_steam_plants_fuel_by_plant_sched402", 26_947), ("out_ferc1__yearly_steam_plants_fuel_sched402", 51_238), ("out_ferc1__yearly_steam_plants_sched402", 32_975), - ("out_ferc1__yearly_transmission_lines_sched422", 640_619), # unverified - ("out_ferc1__yearly_utility_plant_summary_sched200", 198_769), # unverified + ("out_ferc1__yearly_transmission_lines_sched422", 640_619), + ("out_ferc1__yearly_utility_plant_summary_sched200", 198_769), ("out_ferc1__yearly_small_plants_sched410", 17_763), ], ) @@ -130,8 +125,7 @@ def test_minmax_rows(live_dbs, asset_value_loader, expected_rows, asset_key): live_dbs: Boolean (wether we're using a live or testing DB). expected_rows (int): Expected number of rows that the dataframe should contain when all data is loaded and is output without aggregation. - asset_key (str): Shorthand name identifying the dataframe, corresponding - to the name of the asset. + asset_key (str): The name of the asset. """ if not live_dbs: pytest.skip("Data validation only works with a live PUDL DB.")