Merge pull request #3073 from robertozanchi/eia923-2022-final-release-q4

Eia923 2022 final release q4 update Nov 21
catalyst-cooperative · Nov 29, 2023 · b2b8339 · b2b8339
2 parents 4fc35e7 + 2994731
commit b2b8339
Show file tree

Hide file tree

Showing 9 changed files with 43 additions and 44 deletions.
diff --git a/.github/workflows/zenodo-cache-sync.yml b/.github/workflows/zenodo-cache-sync.yml
@@ -41,8 +41,6 @@ jobs:
 
       - name: Checkout desired branch
         uses: actions/checkout@v4
-        with:
-          ref: ${{ env.GITHUB_REF }}
 
       - name: Install Conda environment using mamba
         uses: mamba-org/setup-micromamba@v1

diff --git a/README.rst b/README.rst
@@ -52,11 +52,11 @@ What data is available?
 
 PUDL currently integrates data from:
 
-* `EIA Form 860 <https://www.eia.gov/electricity/data/eia860/>`__: 2001-2022
+* `EIA Form 860 <https://www.eia.gov/electricity/data/eia860/>`__: 2001 - 2022
 * `EIA Form 860m <https://www.eia.gov/electricity/data/eia860m/>`__: 2023-06
-* `EIA Form 861 <https://www.eia.gov/electricity/data/eia861/>`__: 2001-2022
-* `EIA Form 923 <https://www.eia.gov/electricity/data/eia923/>`__: 2001-2022
-* `EPA Continuous Emissions Monitoring System (CEMS) <https://campd.epa.gov/>`__: 1995-2022
+* `EIA Form 861 <https://www.eia.gov/electricity/data/eia861/>`__: 2001 - 2022
+* `EIA Form 923 <https://www.eia.gov/electricity/data/eia923/>`__: 2001 - 2023-08
+* `EPA Continuous Emissions Monitoring System (CEMS) <https://campd.epa.gov/>`__: 1995 - 2022
 * `FERC Form 1 <https://www.ferc.gov/industries-data/electric/general-information/electric-industry-forms/form-1-electric-utility-annual>`__: 1994-2021
 * `FERC Form 714 <https://www.ferc.gov/industries-data/electric/general-information/electric-industry-forms/form-no-714-annual-electric/data>`__: 2006-2020
 * `US Census Demographic Profile 1 Geodatabase <https://www.census.gov/geographies/mapping-files/2010/geo/tiger-data.html>`__: 2010

diff --git a/docs/release_notes.rst b/docs/release_notes.rst
@@ -72,8 +72,8 @@ Data Coverage
 
 * Updated :doc:`data_sources/eia860` to include final release data from 2022.
 * Updated :doc:`data_sources/eia861` to include final release data from 2022.
-* Updated :doc:`data_sources/eia923` to include early release data from 2022 and
-  monthly YTD data as of April 2023.
+* Updated :doc:`data_sources/eia923` to include final release data from 2022 and
+  monthly YTD data as of October 2023.
 * Updated :doc:`data_sources/epacems` to switch from the old FTP server to the new
   CAMPD API, and to include 2022 data. Due to changes in the ETL, Alaska, Puerto Rico
   and Hawaii are now included in CEMS processing. See issue :issue:`1264` & PRs

diff --git a/docs/templates/eia923_child.rst.jinja b/docs/templates/eia923_child.rst.jinja
@@ -35,7 +35,8 @@ in `EIA Form 423
 replaced the earlier FERC Form 423). If you're interested in this earlier data, get in
 touch with us!
 
-Monthly interim EIA-923 data releases are not yet integrated into PUDL. In addition, We
+Monthly interim EIA-923 data are periodically integrated into PUDL as well. Incomplete
+year-to-date data are excluded from the annualized tables to avoid confusion. We
 have not yet integrated tables reporting fuel stocks, data from Puerto Rico, or EIA-923
 schedules 6, 7, and 8.
 {% endblock %}

diff --git a/src/pudl/package_data/eia923/file_map.csv b/src/pudl/package_data/eia923/file_map.csv
diff --git a/src/pudl/package_data/eia923/skiprows.csv b/src/pudl/package_data/eia923/skiprows.csv
@@ -1,14 +1,14 @@
 year_index,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023
-generation_fuel,7,7,7,7,7,7,7,7,7,7,5,5,5,5,5,5,5,5,5,5,5,6,5
-puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,5,5,6,6,6,7,5
-stocks,7,7,7,7,7,7,7,7,7,7,5,5,5,5,5,5,5,5,5,5,5,6,4
-oil_stocks,-1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,4
+generation_fuel,7,7,7,7,7,7,7,7,7,7,5,5,5,5,5,5,5,5,5,5,5,5,5
+puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,5,5,6,6,6,6,5
+stocks,7,7,7,7,7,7,7,7,7,7,5,5,5,5,5,5,5,5,5,5,5,5,4
+oil_stocks,-1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,4
 coal_stocks,-1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,-1,-1
 petcoke_stocks,-1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,-1,-1
-energy_storage,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,5,5,5,5,6,4
-boiler_fuel,-1,-1,-1,-1,-1,-1,-1,7,7,7,5,5,5,5,5,5,5,5,5,5,5,6,4
-generator,-1,-1,-1,-1,-1,-1,-1,7,7,7,5,5,5,5,5,5,5,5,5,5,5,6,4
-fuel_receipts_costs,-1,-1,-1,-1,-1,-1,-1,7,6,7,4,4,4,4,4,4,4,4,4,4,4,5,3
-plant_frame,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,4,4,4,4,4,4,4,5,3
-plant_frame_puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,5,3
-emissions_control,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,4,4,4,4,4,4,5,-1
+energy_storage,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,5,5,5,5,5,4
+boiler_fuel,-1,-1,-1,-1,-1,-1,-1,7,7,7,5,5,5,5,5,5,5,5,5,5,5,5,4
+generator,-1,-1,-1,-1,-1,-1,-1,7,7,7,5,5,5,5,5,5,5,5,5,5,5,5,4
+fuel_receipts_costs,-1,-1,-1,-1,-1,-1,-1,7,6,7,4,4,4,4,4,4,4,4,4,4,4,4,3
+plant_frame,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,4,4,4,4,4,4,4,4,3
+plant_frame_puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,3
+emissions_control,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,4,4,4,4,4,4,4,-1
diff --git a/src/pudl/workspace/datastore.py b/src/pudl/workspace/datastore.py
@@ -174,7 +174,7 @@ class ZenodoDoiSettings(BaseSettings):
     eia860: ZenodoDoi = "10.5281/zenodo.10067566"
     eia860m: ZenodoDoi = "10.5281/zenodo.10204686"
     eia861: ZenodoDoi = "10.5281/zenodo.10204708"
-    eia923: ZenodoDoi = "10.5281/zenodo.8172818"
+    eia923: ZenodoDoi = "10.5281/zenodo.10067550"
     eia_bulk_elec: ZenodoDoi = "10.5281/zenodo.7067367"
     epacamd_eia: ZenodoDoi = "10.5281/zenodo.7900974"
     epacems: ZenodoDoi = "10.5281/zenodo.8235497"

diff --git a/test/validate/eia_test.py b/test/validate/eia_test.py
@@ -45,15 +45,15 @@ def test_no_null_cols_eia(pudl_out_eia, live_dbs, cols, df_name):
 @pytest.mark.parametrize(
     "df_name,raw_rows,monthly_rows,annual_rows",
     [
-        ("bf_eia923", 1_559_257, 1_559_257, 127_412),
-        ("bga_eia860", 141_652, 141_652, 141_652),
-        ("boil_eia860", 83_356, 83_356, 83_356),
-        ("frc_eia923", 639_647, 261_583, 25_370),
-        ("gen_eia923", None, 5_179_377, 433_332),
-        ("gens_eia860", 556_948, 556_948, 556_948),
-        ("gf_eia923", 2_879_884, 2_879_884, 244_795),
+        ("bf_eia923", 1_569_568, 1_569_568, 128_252),
+        ("bga_eia860", 142_391, 142_391, 142_391),
+        ("boil_eia860", 83_416, 83_416, 83_416),
+        ("frc_eia923", 646_677, 264_043, 25_443),
+        ("gen_eia923", None, 5_179_478, 433_336),
+        ("gens_eia860", 556_949, 556_949, 556_949),
+        ("gf_eia923", 2_907_735, 2_907_735, 246_324),
         ("own_eia860", 89_741, 89_741, 89_741),
-        ("plants_eia860", 200_514, 200_514, 200_514),
+        ("plants_eia860", 200_511, 200_511, 200_511),
         ("pu_eia860", 199_635, 199_635, 199_635),
         ("utils_eia860", 139_883, 139_883, 139_883),
         ("emissions_control_equipment_eia860", 56_616, 56_616, 56_616),

diff --git a/test/validate/mcoe_test.py b/test/validate/mcoe_test.py
@@ -106,11 +106,11 @@ def test_no_null_rows_mcoe(pudl_out_mcoe, live_dbs, df_name, thresh):
 @pytest.mark.parametrize(
     "df_name,monthly_rows,annual_rows",
     [
-        ("hr_by_unit", 387_670, 32_414),
-        ("hr_by_gen", 599_496, 50_070),
-        ("fuel_cost", 599_496, 50_070),
-        ("capacity_factor", 5_179_377, 433_332),
-        ("mcoe", 5_179_785, 433_366),
+        ("hr_by_unit", 389_530, 32_569),
+        ("hr_by_gen", 602_580, 50_327),
+        ("fuel_cost", 602_580, 50_327),
+        ("capacity_factor", 5_179_478, 433_336),
+        ("mcoe", 5_179_886, 433_370),
     ],
 )
 def test_minmax_rows_mcoe(pudl_out_mcoe, live_dbs, monthly_rows, annual_rows, df_name):