update boolean columns associated with short-form table

migrations/versions/49387e87e70a_wipe_and_reset.py

@@ -589,7 +589,7 @@ def upgrade() -> None:
     sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'),
     sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'),
     sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'),
-    sa.Column('water_heater', sa.Integer(), nullable=True, comment="The number of grid-enabled water heaters added to the respondent's program this year - if the respondent has DSM program for grid-enabled water heaters (as defined by DOE’s Office of Energy Efficiency and Renewable Energy)."),
+    sa.Column('num_water_heaters', sa.Integer(), nullable=True, comment="The number of grid-enabled water heaters added to the respondent's program this year - if the respondent has DSM program for grid-enabled water heaters (as defined by DOE’s Office of Energy Efficiency and Renewable Energy)."),
     sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'),
     sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_demand_response_water_heater_data_maturity_core_pudl__codes_data_maturities')),
     sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_core_eia861__yearly_demand_response_water_heater'))
@@ -632,7 +632,7 @@ def upgrade() -> None:
     sa.Column('reported_as_another_company', sa.Text(), nullable=True, comment="The name of the company if a respondent's demand-side management activities are reported on Schedule 6 of another company’s form."),
     sa.Column('short_form', sa.Boolean(), nullable=True, comment='Whether the reported information comes from the short form. In the case of form EIA 861, a shorter version of the form was created in 2012 to reduce respondent burden on smaller utilities and increase our processing efficiency.'),
     sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'),
-    sa.Column('time_responsive_programs', sa.Boolean(), nullable=True, comment='Whether the respondent operates any time-based rate programs (e.g., real-time pricing, critical peak pricing, variable peak pricing and time-of-use rates administered through a tariff).'),
+    sa.Column('has_time_responsive_programs', sa.Boolean(), nullable=True, comment='Whether the respondent operates any time-based rate programs (e.g., real-time pricing, critical peak pricing, variable peak pricing and time-of-use rates administered through a tariff).'),
     sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'),
     sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'),
     sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'),
@@ -1361,7 +1361,7 @@ def upgrade() -> None:
     sa.Column('natural_gas_pipeline_name_2', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'),
     sa.Column('natural_gas_pipeline_name_3', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'),
     sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'),
-    sa.Column('net_metering', sa.Boolean(), nullable=True, comment='Did this plant have a net metering agreement in effect during the reporting year?  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'),
+    sa.Column('has_net_metering', sa.Boolean(), nullable=True, comment='Did this plant have a net metering agreement in effect during the reporting year?  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'),
     sa.Column('pipeline_notes', sa.Text(), nullable=True, comment='Additional owner or operator of natural gas pipeline.'),
     sa.Column('primary_purpose_id_naics', sa.Integer(), nullable=True, comment='North American Industry Classification System (NAICS) code that best describes the primary purpose of the reporting plant'),
     sa.Column('regulatory_status_code', sa.Text(), nullable=True, comment='Indicates whether the plant is regulated or non-regulated.'),
@@ -2987,7 +2987,7 @@ def upgrade() -> None:
     sa.Column('natural_gas_pipeline_name_2', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'),
     sa.Column('natural_gas_pipeline_name_3', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'),
     sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'),
-    sa.Column('net_metering', sa.Boolean(), nullable=True, comment='Did this plant have a net metering agreement in effect during the reporting year?  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'),
+    sa.Column('has_net_metering', sa.Boolean(), nullable=True, comment='Did this plant have a net metering agreement in effect during the reporting year?  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'),
     sa.Column('pipeline_notes', sa.Text(), nullable=True, comment='Additional owner or operator of natural gas pipeline.'),
     sa.Column('primary_purpose_id_naics', sa.Integer(), nullable=True, comment='North American Industry Classification System (NAICS) code that best describes the primary purpose of the reporting plant'),
     sa.Column('regulatory_status_code', sa.Text(), nullable=True, comment='Indicates whether the plant is regulated or non-regulated.'),

src/pudl/metadata/fields.py

@@ -894,7 +894,7 @@
     },
     "has_demand_side_management": {
         "type": "boolean",
-        "description": "Were there strategies or measures used to control electricity demand by customers?",
+        "description": "Whether there were strategies or measures used to control electricity demand by customers",
     },
     "depreciation_type": {
         "type": "string",
@@ -1719,9 +1719,9 @@
         "type": "string",
         "description": "Category of geographic aggregation in EIA bulk electricity data.",
     },
-    "green_pricing": {
+    "has_green_pricing": {
         "type": "boolean",
-        "description": "Was there green pricing program associated with this utility during the reporting year?",
+        "description": "Whether a green pricing program was associated with this utility during the reporting year.",
     },
     "green_pricing_revenue": {
         "type": "number",
@@ -2297,9 +2297,9 @@
         "description": "Net output for load (net generation - energy used for pumping) in megawatt-hours.",
         "unit": "MWh",
     },
-    "net_metering": {
+    "has_net_metering": {
         "type": "boolean",
-        "description": "Did this plant have a net metering agreement in effect during the reporting year?  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015",
+        "description": "Whether the plant has a net metering agreement in effect during the reporting year.  (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015",
         # TODO: Is this really boolean? Or do we have non-null strings that mean False?
     },
     "net_power_exchanged_mwh": {
@@ -3881,7 +3881,7 @@
             "and predefined, based on season, day of week, and time of day."
         ),
     },
-    "time_responsive_programs": {
+    "has_time_responsive_programs": {
         "type": "boolean",
         "description": (
             "Whether the respondent operates any time-based rate programs (e.g., "
@@ -4189,7 +4189,7 @@
         "unit": "MMBtu_per_hour",
         "description": "Design waste-heat input rate at maximum continuous steam flow where a waste-heat boiler is a boiler that receives all or a substantial portion of its energy input from the noncumbustible exhaust gases of a separate fuel-burning process (MMBtu per hour).",
     },
-    "water_heater": {
+    "num_water_heaters": {
         "type": "integer",
         "description": (
             "The number of grid-enabled water heaters added to the respondent's "

src/pudl/metadata/resources/__init__.py

@@ -56,7 +56,7 @@
             "natural_gas_pipeline_name_2",
             "natural_gas_pipeline_name_3",
             "nerc_region",
-            "net_metering",
+            "has_net_metering",
             "pipeline_notes",
             "primary_purpose_id_naics",
             "regulatory_status_code",

src/pudl/metadata/resources/eia.py

@@ -814,7 +814,7 @@
                 "natural_gas_pipeline_name_2",
                 "natural_gas_pipeline_name_3",
                 "nerc_region",
-                "net_metering",
+                "has_net_metering",
                 "pipeline_notes",
                 "primary_purpose_id_naics",
                 "regulatory_status_code",

src/pudl/metadata/resources/eia860.py

@@ -351,7 +351,7 @@
                 "natural_gas_pipeline_name_2",
                 "natural_gas_pipeline_name_3",
                 "nerc_region",
-                "net_metering",
+                "has_net_metering",
                 "pipeline_notes",
                 "primary_purpose_id_naics",
                 "regulatory_status_code",

src/pudl/metadata/resources/eia861.py

@@ -124,7 +124,7 @@
                 "report_date",
                 "state",
                 "utility_id_eia",
-                "water_heater",
+                "num_water_heaters",
                 "data_maturity",
             ],
             "primary_key": [
@@ -213,7 +213,7 @@
                 "reported_as_another_company",
                 "short_form",
                 "state",
-                "time_responsive_programs",
+                "has_time_responsive_programs",
                 "utility_id_eia",
                 "utility_name_eia",
                 "data_maturity",
@@ -707,10 +707,10 @@
                 "sales_revenue",
                 "sales_mwh",
                 "customers",
-                "water_heater",
-                "net_metering",
+                "num_water_heaters",
+                "has_net_metering",
                 "demand_side_management",
-                "time_responsive_programs",
+                "has_time_responsive_programs",
                 "green_pricing",
                 "data_maturity",
             ],

src/pudl/package_data/eia860/column_maps/plant.csv

@@ -20,7 +20,7 @@ sector_name_eia,,,,,,,,,sector_name,sector_name,sector_name,sector_name,sector_n
 sector_id_eia,,,,,,,,,sector_number,sector_number,sector,sector,sector,sector,sector,sector,sector,sector,sector,sector,sector,sector
 ferc_cogen_status,,,,,,,ferc_cogen,ferc_cogen,ferc_cogen,ferc_cogen,ferc_cogen,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status,ferc_cogeneration_status
 ferc_cogen_docket_no,,,,,,,ferc_cogen_docket,ferc_cogen_docket,ferc_cogen_docket,ferc_cogen_docket,ferc_cogen_docket,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number,ferc_cogeneration_docket_number
-net_metering,,,,,,,,,,,,,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,,,,,,,
+has_net_metering,,,,,,,,,,,,,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,net_metering_for_facilities_with_solar_or_wind_generation,,,,,,,
 ferc_small_power_producer,,,,,,,ferc_small_power,ferc_small_power,ferc_small_power,ferc_small_power,ferc_small_power,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status,ferc_small_power_producer_status
 ferc_small_power_producer_docket_no,,,,,,,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_docket,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number,ferc_small_power_producer_docket_number
 ferc_exempt_wholesale_generator,,,,,,,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status,ferc_exempt_wholesale_generator_status

src/pudl/package_data/eia861/column_maps/demand_response_eia861.csv

@@ -35,5 +35,5 @@ commercial_other_costs,30,30,30,30,31,31,32,31,31,31
 industrial_other_costs,31,31,31,31,32,32,33,32,32,32
 transportation_other_costs,32,32,32,32,33,33,34,33,33,33
 total_other_costs,33,33,33,33,34,34,35,34,34,34
-water_heater,34,34,34,34,35,35,36,35,35,35
+num_water_heaters,34,34,34,34,35,35,36,35,35,35
 early_release,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1

src/pudl/package_data/eia861/column_maps/demand_side_management_eia861.csv

@@ -85,7 +85,7 @@ commercial_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,59,59,59,93,93,87
 industrial_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,60,60,60,94,94,88
 transportation_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,95,95,89
 other_price_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,61,61,61,-1,-1,-1
-time_responsive_programs,-1,-1,-1,-1,-1,-1,-1,62,62,62,96,96,-1
+has_time_responsive_programs,-1,-1,-1,-1,-1,-1,-1,62,62,62,96,96,-1
 residential_time_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,63,63,63,97,97,90
 commercial_time_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,64,64,64,98,98,91
 industrial_time_responsiveness_customers,-1,-1,-1,-1,-1,-1,-1,65,65,65,99,99,92

src/pudl/package_data/eia861/column_maps/frame_eia861.csv

@@ -13,7 +13,7 @@ distribution_systems,8,9,9,9,10,10,10
 dynamic_pricing,9,10,10,10,11,11,11
 energy_efficiency,10,11,11,11,12,12,12
 mergers,11,12,12,12,13,13,13
-net_metering,12,13,13,13,14,14,14
+has_net_metering,12,13,13,13,14,14,14
 non_net_metering_distributed,13,14,14,14,15,15,15
 operational_data,14,15,15,15,16,16,16
 reliability,15,16,16,16,17,17,17

src/pudl/package_data/eia861/column_maps/short_form_eia861.csv

@@ -8,9 +8,9 @@ balancing_authority_code_eia,-1,4,4,5,5,5,5,-1,5,5,5
 sales_revenue,4,5,5,6,6,6,6,-1,6,6,6
 sales_mwh,5,6,6,7,7,7,7,-1,7,7,7
 customers,6,7,7,8,8,8,8,-1,8,8,8
-water_heater,-1,8,8,9,9,9,9,-1,9,9,9
-net_metering,8,9,9,10,10,10,10,-1,10,10,10
-demand_side_management,9,10,10,11,11,11,11,-1,11,11,11
-time_responsive_programs,10,11,11,12,12,12,12,-1,12,12,12
-green_pricing,7,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
+num_water_heaters,-1,8,8,9,9,9,9,-1,9,9,9
+has_net_metering,8,9,9,10,10,10,10,-1,10,10,10
+has_demand_side_management,9,10,10,11,11,11,11,-1,11,11,11
+has_time_responsive_programs,10,11,11,12,12,12,12,-1,12,12,12
+has_green_pricing,7,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
 early_release,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1

src/pudl/transform/eia860.py

@@ -545,7 +545,7 @@ def _core_eia860__plants(raw_eia860__plant: pd.DataFrame) -> pd.DataFrame:
         "energy_storage",
         "natural_gas_storage",
         "liquefied_natural_gas_storage",
-        "net_metering",
+        "has_net_metering",
     ]
 
     for column in boolean_columns_to_fix:

src/pudl/transform/eia861.py

@@ -1234,10 +1234,9 @@ def core_eia861__yearly_short_form(
     ]
 
     bool_cols = [
-        "water_heater",
-        "net_metering",
+        "has_net_metering",
         "demand_side_management",
-        "time_responsive_programs",
+        "has_time_responsive_programs",
         "green_pricing",
     ]
 
@@ -1338,11 +1337,11 @@ def core_eia861__yearly_demand_response(raw_eia861__demand_response: pd.DataFram
     raw_dr["short_form"] = _make_yn_bool(raw_dr.short_form)
 
     # Split data into tidy-able and not
-    raw_dr_water_heater = raw_dr[idx_cols + ["water_heater", "data_maturity"]].copy()
+    raw_dr_water_heater = raw_dr[idx_cols + ["num_water_heaters", "data_maturity"]].copy()
     dr_water = _drop_dupes(
         df=raw_dr_water_heater, df_name="Demand Response Water Heater", subset=idx_cols
     )
-    raw_dr = raw_dr.drop(columns=["water_heater"])
+    raw_dr = raw_dr.drop(columns=["num_water_heaters"])
 
     ###########################################################################
     # Tidy Data:
@@ -1444,7 +1443,7 @@ def core_demand_side_management_eia861(
         "major_program_changes",
         "price_responsive_programs",
         "short_form",
-        "time_responsive_programs",
+        "has_time_responsive_programs",
     ]
 
     cost_cols = [