Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make core ferc1 asset names plural and rename exploded ferc1 assets #2914

Closed
wants to merge 2 commits into from
Closed

Make core ferc1 asset names plural and rename exploded ferc1 assets #2914

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
799282a
Rename a few ferc1 core assets to be plural entities and rename explo…
bendnorman Oct 3, 2023
4260c2e
Rename `heat_rate_mmbtu_mwh -> heat_rate_mmbtu_mwh_by_unit` (#2917)
bendnorman Oct 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/release_notes.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ Data Coverage
:pr:`2134`.
* :ref:`core_ferc1__yearly_depreciation_amortization_summary`, see issue :issue:`1816`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

depreciation_amortization_summaries?

& PR :pr:`2143`.
* :ref:`core_ferc1__yearly_income_statement`, see issue :issue:`1813` & PR :pr:`2147`.
* :ref:`core_ferc1__yearly_income_statements`, see issue :issue:`1813` & PR
:pr:`2147`.
* :ref:`core_ferc1__yearly_electric_plant_depreciation_changes` see issue
:issue:`1808` & :pr:`2119`.
* :ref:`core_ferc1__yearly_electric_plant_depreciation_functional` see issue
Expand Down Expand Up @@ -223,7 +224,7 @@ Data Cleaning
* Fixed column naming issues in the
:ref:`core_ferc1__yearly_electric_operating_revenues` table.
* Made minor calculation fixes in the metadata for
:ref:`core_ferc1__yearly_income_statement`,
:ref:`core_ferc1__yearly_income_statements`,
:ref:`core_ferc1__yearly_utility_plant_summary`,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

plant_summaries by this logic, or no?

:ref:`core_ferc1__yearly_electric_operating_revenues`,
:ref:`core_ferc1__yearly_balance_sheet_assets`,
Expand Down
355 changes: 355 additions & 0 deletions migrations/versions/ab0540a6484c_rename_core_ferc1_assets_to_be_plural.py
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/pudl/analysis/classify_plants_ferc1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,7 @@ def fuel_by_plant_ferc1(

Returns:
DataFrame with a single record for each plant-year, including the columns
required to merge it with the core_ferc1__yearly_plants_steam table/DataFrame (report_year,
required to merge it with the core_ferc1__yearly_steam_plants table/DataFrame (report_year,
utility_id_ferc1, and plant_name) as well as totals for fuel mmbtu consumed in
that plant-year, and the cost of fuel in that year, the proportions of heat
content and fuel costs for each fuel in that year, and a column that labels the
Expand Down
10 changes: 5 additions & 5 deletions src/pudl/extract/ferc1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,19 +104,19 @@
"dbf": "f1_fuel",
"xbrl": "steam_electric_generating_plant_statistics_large_plants_fuel_statistics_402",
},
"core_ferc1__yearly_plants_steam": {
"core_ferc1__yearly_steam_plants": {
"dbf": "f1_steam",
"xbrl": "steam_electric_generating_plant_statistics_large_plants_402",
},
"core_ferc1__yearly_plants_small": {
"core_ferc1__yearly_small_plants": {
"dbf": "f1_gnrt_plant",
"xbrl": "generating_plant_statistics_410",
},
"core_ferc1__yearly_plants_hydro": {
"core_ferc1__yearly_hydro_plants": {
"dbf": "f1_hydro",
"xbrl": "hydroelectric_generating_plant_statistics_large_plants_406",
},
"core_ferc1__yearly_plants_pumped_storage": {
"core_ferc1__yearly_pumped_storage_plants": {
"dbf": "f1_pumped_storage",
"xbrl": "pumped_storage_generating_plant_statistics_large_plants_408",
},
Expand Down Expand Up @@ -156,7 +156,7 @@
"dbf": "f1_comp_balance_db",
"xbrl": "comparative_balance_sheet_assets_and_other_debits_110",
},
"core_ferc1__yearly_income_statement": {
"core_ferc1__yearly_income_statements": {
"dbf": ["f1_income_stmnt", "f1_incm_stmnt_2"],
"xbrl": "statement_of_income_114",
},
Expand Down
8 changes: 4 additions & 4 deletions src/pudl/glue/ferc1_eia.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,10 +255,10 @@ def get_plants_ferc1_raw_job() -> JobDefinition:
This job expects ferc1.sqlite and ferc_xbrl.sqlite databases to be populated.
"""
plant_tables = [
"core_ferc1__yearly_plants_hydro",
"core_ferc1__yearly_plants_small",
"core_ferc1__yearly_plants_pumped_storage",
"core_ferc1__yearly_plants_steam",
"core_ferc1__yearly_hydro_plants",
"core_ferc1__yearly_small_plants",
"core_ferc1__yearly_pumped_storage_plants",
"core_ferc1__yearly_steam_plants",
"core_ferc1__yearly_fuel", # bc it has plants/is associated w/ the steam table
]

Expand Down
4 changes: 2 additions & 2 deletions src/pudl/metadata/fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2817,7 +2817,7 @@

FIELD_METADATA_BY_RESOURCE: dict[str, dict[str, Any]] = {
"sector_consolidated_eia": {"code": {"type": "integer"}},
"core_ferc1__yearly_plants_hydro": {
"core_ferc1__yearly_hydro_plants": {
"plant_type": {
"type": "string",
"constraints": {
Expand All @@ -2835,7 +2835,7 @@
},
}
},
"core_ferc1__yearly_plants_steam": {
"core_ferc1__yearly_steam_plants": {
"plant_type": {
"type": "string",
"constraints": {
Expand Down
10 changes: 5 additions & 5 deletions src/pudl/metadata/resources/ferc1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@
"etl_group": "ferc1",
"field_namespace": "ferc1",
},
"core_ferc1__yearly_income_statement": {
"core_ferc1__yearly_income_statements": {
"description": "Statement of Income. Schedule 114.",
"schema": {
"fields": [
Expand Down Expand Up @@ -432,7 +432,7 @@
"etl_group": "glue",
"field_namespace": "ferc1",
},
"core_ferc1__yearly_plants_hydro": {
"core_ferc1__yearly_hydro_plants": {
"description": (
"Hydroelectric generating plant statistics for large plants. Large plants "
"have an installed nameplate capacity of more than 10 MW. As reported on "
Expand Down Expand Up @@ -484,7 +484,7 @@
"etl_group": "ferc1",
"field_namespace": "ferc1",
},
"core_ferc1__yearly_plants_pumped_storage": {
"core_ferc1__yearly_pumped_storage_plants": {
"description": (
"Generating plant statistics for hydroelectric pumped storage plants with "
"an installed nameplate capacity of 10+ MW. As reported in Scheudle 408 of "
Expand Down Expand Up @@ -540,7 +540,7 @@
"etl_group": "ferc1",
"field_namespace": "ferc1",
},
"core_ferc1__yearly_plants_small": {
"core_ferc1__yearly_small_plants": {
"description": """The generating plant statistics for internal combustion
plants, gas turbine-plants, conventional hydro plants, and pumped storage plants with
less than 10 MW installed nameplate capacity and steam plants with less than 25 MW
Expand Down Expand Up @@ -581,7 +581,7 @@
"etl_group": "ferc1",
"field_namespace": "ferc1",
},
"core_ferc1__yearly_plants_steam": {
"core_ferc1__yearly_steam_plants": {
"description": (
"Generating plant statistics for steam plants with a capacity of 25+ MW, "
"internal combustion and gas-turbine plants of 10+ MW, and all nuclear "
Expand Down
43 changes: 23 additions & 20 deletions src/pudl/output/ferc1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class CalculationToleranceFerc1(BaseModel):


EXPLOSION_CALCULATION_TOLERANCES: dict[str, CalculationToleranceFerc1] = {
"core_ferc1__yearly_income_statement": CalculationToleranceFerc1(
"core_ferc1__yearly_income_statements": CalculationToleranceFerc1(
intertable_calculation_errors=0.20,
),
"core_ferc1__yearly_balance_sheet_assets": CalculationToleranceFerc1(
Expand Down Expand Up @@ -68,7 +68,7 @@ def _out_ferc1__yearly_plants_utilities(
@asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python")
def _out_ferc1__yearly_steam_plants(
_out_ferc1__yearly_plants_utilities: pd.DataFrame,
core_ferc1__yearly_plants_steam: pd.DataFrame,
core_ferc1__yearly_steam_plants: pd.DataFrame,
) -> pd.DataFrame:
"""Select and joins some useful fields from the FERC Form 1 steam table.

Expand All @@ -81,13 +81,13 @@ def _out_ferc1__yearly_steam_plants(
Args:
_out_ferc1__yearly_plants_utilities: Denormalized dataframe of FERC Form 1 plants and
utilities data.
core_ferc1__yearly_plants_steam: The normalized FERC Form 1 steam table.
core_ferc1__yearly_steam_plants: The normalized FERC Form 1 steam table.

Returns:
A DataFrame containing useful fields from the FERC Form 1 steam table.
"""
steam_df = (
core_ferc1__yearly_plants_steam.merge(
core_ferc1__yearly_steam_plants.merge(
_out_ferc1__yearly_plants_utilities,
on=["utility_id_ferc1", "plant_name_ferc1"],
how="left",
Expand Down Expand Up @@ -122,12 +122,12 @@ def _out_ferc1__yearly_steam_plants(

@asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python")
def _out_ferc1__yearly_small_plants(
core_ferc1__yearly_plants_small: pd.DataFrame,
core_ferc1__yearly_small_plants: pd.DataFrame,
_out_ferc1__yearly_plants_utilities: pd.DataFrame,
) -> pd.DataFrame:
"""Pull a useful dataframe related to the FERC Form 1 small plants."""
plants_small_df = (
core_ferc1__yearly_plants_small.merge(
core_ferc1__yearly_small_plants.merge(
_out_ferc1__yearly_plants_utilities,
on=["utility_id_ferc1", "plant_name_ferc1"],
how="left",
Expand Down Expand Up @@ -159,12 +159,12 @@ def _out_ferc1__yearly_small_plants(

@asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python")
def _out_ferc1__yearly_hydro_plants(
core_ferc1__yearly_plants_hydro: pd.DataFrame,
core_ferc1__yearly_hydro_plants: pd.DataFrame,
_out_ferc1__yearly_plants_utilities: pd.DataFrame,
) -> pd.DataFrame:
"""Pull a useful dataframe related to the FERC Form 1 hydro plants."""
plants_hydro_df = (
core_ferc1__yearly_plants_hydro.merge(
core_ferc1__yearly_hydro_plants.merge(
_out_ferc1__yearly_plants_utilities,
on=["utility_id_ferc1", "plant_name_ferc1"],
how="left",
Expand All @@ -190,12 +190,12 @@ def _out_ferc1__yearly_hydro_plants(

@asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python")
def _out_ferc1__yearly_pumped_storage_plants(
core_ferc1__yearly_plants_pumped_storage: pd.DataFrame,
core_ferc1__yearly_pumped_storage_plants: pd.DataFrame,
_out_ferc1__yearly_plants_utilities: pd.DataFrame,
) -> pd.DataFrame:
"""Pull a dataframe of FERC Form 1 Pumped Storage plant data."""
pumped_storage_df = (
core_ferc1__yearly_plants_pumped_storage.merge(
core_ferc1__yearly_pumped_storage_plants.merge(
_out_ferc1__yearly_plants_utilities,
on=["utility_id_ferc1", "plant_name_ferc1"],
how="left",
Expand Down Expand Up @@ -578,11 +578,11 @@ def out_ferc1__yearly_electricity_sales_by_rate_schedule(

@asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python")
def out_ferc1__yearly_income_statement(
core_ferc1__yearly_income_statement: pd.DataFrame,
core_ferc1__yearly_income_statements: pd.DataFrame,
core_pudl__assn_utilities_ferc1: pd.DataFrame,
) -> pd.DataFrame:
"""Pull a useful dataframe of FERC Form 1 Purchased Power data."""
out_ferc1__yearly_income_statement = core_ferc1__yearly_income_statement.merge(
out_ferc1__yearly_income_statement = core_ferc1__yearly_income_statements.merge(
core_pudl__assn_utilities_ferc1, on="utility_id_ferc1"
).pipe(
pudl.helpers.organize_cols,
Expand Down Expand Up @@ -1037,7 +1037,10 @@ def exploded_table_asset_factory(
}
ins |= {table_name: AssetIn(table_name) for table_name in table_names_to_explode}

@asset(name=f"exploded_{root_table}", ins=ins, io_manager_key=io_manager_key)
asset_group, asset_name = root_table.split("__")
asset_name = f"{asset_group}__exploded_{asset_name}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is exploded the asset type in this model? Seems like something might be missing here, possibly yearly?


@asset(name=asset_name, ins=ins, io_manager_key=io_manager_key)
def exploded_tables_asset(
**kwargs: dict[str, pd.DataFrame],
) -> pd.DataFrame:
Expand Down Expand Up @@ -1076,19 +1079,19 @@ def create_exploded_table_assets() -> list[AssetsDefinition]:
"""
explosion_args = [
{
"root_table": "income_statement_ferc1",
"root_table": "core_ferc1__yearly_income_statements",
"table_names_to_explode": [
"core_ferc1__yearly_income_statement",
"core_ferc1__yearly_income_statements",
"core_ferc1__yearly_depreciation_amortization_summary",
"core_ferc1__yearly_electric_operating_expenses",
"core_ferc1__yearly_electric_operating_revenues",
],
"calculation_tolerance": EXPLOSION_CALCULATION_TOLERANCES[
"core_ferc1__yearly_income_statement"
"core_ferc1__yearly_income_statements"
],
"seed_nodes": [
NodeId(
table_name="core_ferc1__yearly_income_statement",
table_name="core_ferc1__yearly_income_statements",
xbrl_factoid="net_income_loss",
utility_type="total",
plant_status=pd.NA,
Expand All @@ -1097,7 +1100,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]:
],
},
{
"root_table": "balance_sheet_assets_ferc1",
"root_table": "core_ferc1__yearly_balance_sheet_assets",
"table_names_to_explode": [
"core_ferc1__yearly_balance_sheet_assets",
"core_ferc1__yearly_balance_sheet_assets",
Expand All @@ -1119,7 +1122,7 @@ def create_exploded_table_assets() -> list[AssetsDefinition]:
],
},
{
"root_table": "balance_sheet_liabilities_ferc1",
"root_table": "core_ferc1__yearly_balance_sheet_liabilities",
"table_names_to_explode": [
"core_ferc1__yearly_balance_sheet_liabilities",
"core_ferc1__yearly_balance_sheet_liabilities",
Expand Down Expand Up @@ -1545,7 +1548,7 @@ class XbrlCalculationForestFerc1(BaseModel):
"""A class for manipulating groups of hierarchically nested XBRL calculations.

We expect that the facts reported in high-level FERC tables like
:ref:`core_ferc1__yearly_income_statement` and
:ref:`core_ferc1__yearly_income_statements` and
:ref:`core_ferc1__yearly_balance_sheet_assets` should be
calculable from many individually reported granular values, based on the
calculations encoded in the XBRL Metadata, and that these relationships should have
Expand Down
Loading