diff --git a/.github/workflows/bot-auto-merge.yml b/.github/workflows/bot-auto-merge.yml index 898775e50a..a8b216a008 100644 --- a/.github/workflows/bot-auto-merge.yml +++ b/.github/workflows/bot-auto-merge.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Impersonate auto merge PR bot - uses: tibdex/github-app-token@v1 + uses: tibdex/github-app-token@v2 id: generate-token with: app_id: ${{ secrets.BOT_AUTO_MERGE_PRS_APP_ID }} diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml index eec4318ecf..1dfa75b644 100644 --- a/.github/workflows/build-deploy-pudl.yml +++ b/.github/workflows/build-deploy-pudl.yml @@ -45,7 +45,7 @@ jobs: - name: Docker Metadata id: docker_metadata - uses: docker/metadata-action@v4.4.0 + uses: docker/metadata-action@v5.3.0 with: images: catalystcoop/pudl-etl flavor: | @@ -83,7 +83,7 @@ jobs: # Setup gcloud CLI - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 - name: Determine commit information run: |- diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index d82ebf74cf..6cc14b8d58 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -17,7 +17,7 @@ jobs: - name: Docker Metadata id: docker_metadata - uses: docker/metadata-action@v4.4.0 + uses: docker/metadata-action@v5.3.0 with: images: catalystcoop/pudl-etl flavor: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a6385c14d..458bac69f4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: with: fetch-depth: 2 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Build source and wheel distributions diff --git a/.github/workflows/run-etl.yml b/.github/workflows/run-etl.yml index d8876e940c..014c3ccb6c 100644 --- a/.github/workflows/run-etl.yml +++ b/.github/workflows/run-etl.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v4 - name: Docker Metadata id: docker_metadata - uses: docker/metadata-action@v4.4.0 + uses: docker/metadata-action@v5.3.0 # TODO(rousik): we could consider YYYY-MM-DD-HHMM-branch-sha with: images: catalystcoop/pudl-etl-ci diff --git a/.github/workflows/zenodo-cache-sync.yml b/.github/workflows/zenodo-cache-sync.yml index f52d74be78..d4d68c115d 100644 --- a/.github/workflows/zenodo-cache-sync.yml +++ b/.github/workflows/zenodo-cache-sync.yml @@ -69,7 +69,7 @@ jobs: service_account: "zenodo-cache-manager@catalyst-cooperative-pudl.iam.gserviceaccount.com" - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 - name: Update GCS cache with any new Zenodo archives run: | diff --git a/environments/conda-linux-64.lock.yml b/environments/conda-linux-64.lock.yml index 1e2f208e53..0dbbdcef09 100644 --- a/environments/conda-linux-64.lock.yml +++ b/environments/conda-linux-64.lock.yml @@ -43,11 +43,11 @@ dependencies: - libbrotlicommon=1.1.0=hd590300_1 - libcrc32c=1.1.2=h9c3ff4c_0 - libdeflate=1.19=hd590300_0 - - libev=4.33=h516909a_1 + - libev=4.33=hd590300_2 - libexpat=2.5.0=hcb278e6_1 - libffi=3.4.2=h7f98852_5 - libgfortran5=13.2.0=ha4646dd_3 - - libiconv=1.17=h166bdaf_0 + - libiconv=1.17=hd590300_1 - libjpeg-turbo=3.0.0=hd590300_1 - libnsl=2.0.1=hd590300_0 - libnuma=2.0.16=h0b41bf4_1 @@ -62,7 +62,7 @@ dependencies: - lzo=2.10=h516909a_1000 - ncurses=6.4=h59595ed_2 - nspr=4.35=h27087fc_0 - - openssl=3.1.4=hd590300_0 + - openssl=3.2.0=hd590300_1 - pixman=0.42.2=h59595ed_0 - pthread-stubs=0.4=h36c2ea0_1001 - rdma-core=49.0=hd3aeb46_1 @@ -92,7 +92,7 @@ dependencies: - libgfortran-ng=13.2.0=h69a702a_3 - libkml=1.3.0=h01aab08_1018 - libllvm14=14.0.6=hcd5def8_4 - - libnghttp2=1.58.0=h47da74e_0 + - libnghttp2=1.58.0=h47da74e_1 - libpng=1.6.39=h753d276_0 - libprotobuf=4.24.4=hf27288f_0 - libre2-11=2023.06.02=h7a70373_0 @@ -160,7 +160,7 @@ dependencies: - colorama=0.4.6=pyhd8ed1ab_0 - crashtest=0.4.1=pyhd8ed1ab_0 - cycler=0.12.1=pyhd8ed1ab_0 - - dagster-pipes=1.5.10=pyhd8ed1ab_0 + - dagster-pipes=1.5.11=pyhd8ed1ab_0 - dataclasses=0.8=pyhc8e2a94_3 - dbus=1.13.6=h5008d03_3 - debugpy=1.8.0=py311hb755f60_1 @@ -181,7 +181,7 @@ dependencies: - fsspec=2023.12.1=pyhca7485f_0 - gdk-pixbuf=2.42.10=h829c605_4 - google-cloud-sdk=456.0.0=py311h38be061_0 - - greenlet=3.0.1=py311hb755f60_0 + - greenlet=3.0.2=py311hb755f60_0 - gts=0.7.6=h977cf35_4 - hpack=4.0.0=pyh9f0ad1d_0 - httptools=0.6.1=py311h459d7ec_0 @@ -202,9 +202,9 @@ dependencies: - kiwisolver=1.4.5=py311h9547e67_1 - lcms2=2.16=hb7c19ff_0 - libblas=3.9.0=20_linux64_openblas - - libcurl=8.4.0=hca28451_0 + - libcurl=8.5.0=hca28451_0 - libgrpc=1.59.2=hd6c4280_0 - - libpq=16.1=hfc447b1_0 + - libpq=16.1=h33b98f1_7 - libwebp=1.3.2=h658648e_1 - llvmlite=0.41.1=py311ha6695c7_0 - locket=1.0.0=pyhd8ed1ab_0 @@ -233,7 +233,7 @@ dependencies: - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 - platformdirs=4.1.0=pyhd8ed1ab_0 - pluggy=1.3.0=pyhd8ed1ab_0 - - prettier=3.1.0=h31abb78_0 + - prettier=3.1.1=h31abb78_0 - prometheus_client=0.19.0=pyhd8ed1ab_0 - psutil=5.9.5=py311h459d7ec_1 - ptyprocess=0.7.0=pyhd3deb0d_0 @@ -287,7 +287,7 @@ dependencies: - traitlets=5.14.0=pyhd8ed1ab_0 - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0 - types-pyyaml=6.0.12.12=pyhd8ed1ab_0 - - typing_extensions=4.8.0=pyha770c72_0 + - typing_extensions=4.9.0=pyha770c72_0 - typing_utils=0.1.0=pyhd8ed1ab_0 - unicodecsv=0.14.1=pyhd8ed1ab_2 - uri-template=1.3.0=pyhd8ed1ab_0 @@ -321,7 +321,7 @@ dependencies: - cached-property=1.5.2=hd8ed1ab_1 - cairo=1.18.0=h3faef2a_0 - cffi=1.16.0=py311hb3a22ac_0 - - cfitsio=4.3.0=hbdc6101_0 + - cfitsio=4.3.1=hbdc6101_0 - click-default-group=1.2.4=pyhd8ed1ab_0 - click-default-group-wheel=1.2.2=pyhd8ed1ab_0 - click-plugins=1.1.1=py_0 @@ -330,16 +330,16 @@ dependencies: - coloredlogs=14.0=pyhd8ed1ab_3 - comm=0.1.4=pyhd8ed1ab_0 - coverage=7.3.2=py311h459d7ec_0 - - curl=8.4.0=hca28451_0 + - curl=8.5.0=hca28451_0 - fonttools=4.46.0=py311h459d7ec_0 - gitdb=4.0.11=pyhd8ed1ab_0 - graphql-core=3.2.3=pyhd8ed1ab_0 - grpcio=1.59.2=py311ha6695c7_0 - h11=0.14.0=pyhd8ed1ab_0 - h2=4.1.0=pyhd8ed1ab_0 - - hdf5=1.14.2=nompi_h4f84152_100 + - hdf5=1.14.3=nompi_h4f84152_100 - html5lib=1.1=pyh9f0ad1d_0 - - hypothesis=6.91.1=pyha770c72_0 + - hypothesis=6.92.0=pyha770c72_0 - importlib-metadata=7.0.0=pyha770c72_0 - importlib_resources=6.1.1=pyhd8ed1ab_0 - isodate=0.6.1=pyhd8ed1ab_0 @@ -365,9 +365,9 @@ dependencies: - partd=1.4.1=pyhd8ed1ab_0 - pexpect=4.8.0=pyh1a96a4e_2 - pillow=10.1.0=py311ha6c5da5_0 - - pint=0.22=pyhd8ed1ab_1 + - pint=0.23=pyhd8ed1ab_0 - pip=23.3.1=pyhd8ed1ab_0 - - postgresql=16.1=h8972f4a_0 + - postgresql=16.1=h8972f4a_7 - proj=9.3.0=h1d62c97_2 - prompt-toolkit=3.0.41=pyha770c72_0 - protobuf=4.24.4=py311h46cbc50_0 @@ -387,7 +387,7 @@ dependencies: - terminado=0.18.0=pyh0d859eb_0 - tinycss2=1.2.1=pyhd8ed1ab_0 - tqdm=4.66.1=pyhd8ed1ab_0 - - typing-extensions=4.8.0=hd8ed1ab_0 + - typing-extensions=4.9.0=hd8ed1ab_0 - typing_inspect=0.9.0=pyhd8ed1ab_0 - universal_pathlib=0.1.4=pyhd8ed1ab_0 - urllib3=1.26.18=pyhd8ed1ab_0 @@ -402,7 +402,7 @@ dependencies: - arrow=1.3.0=pyhd8ed1ab_0 - async-timeout=4.0.3=pyhd8ed1ab_0 - aws-c-s3=0.4.1=hfadff92_0 - - botocore=1.33.10=pyhd8ed1ab_0 + - botocore=1.33.11=pyhd8ed1ab_0 - branca=0.7.0=pyhd8ed1ab_1 - croniter=2.0.1=pyhd8ed1ab_0 - cryptography=41.0.7=py311hcb13ee4_1 @@ -418,7 +418,7 @@ dependencies: - httpcore=1.0.2=pyhd8ed1ab_0 - importlib_metadata=7.0.0=hd8ed1ab_0 - jsonschema-specifications=2023.11.2=pyhd8ed1ab_0 - - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 + - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0 - kealib=1.5.2=hcd42e92_1 - libnetcdf=4.9.2=nompi_h80fb2b6_112 - libspatialite=5.1.0=h7385560_2 @@ -426,7 +426,7 @@ dependencies: - numpy=1.26.2=py311h64a7726_0 - pbr=6.0.0=pyhd8ed1ab_0 - pendulum=2.1.2=py311h459d7ec_6 - - poppler=23.11.0=h590f24d_0 + - poppler=23.12.0=h590f24d_0 - prompt_toolkit=3.0.41=hd8ed1ab_0 - psycopg2-binary=2.9.9=pyhd8ed1ab_0 - pybtex=0.24.0=pyhd8ed1ab_2 @@ -442,7 +442,7 @@ dependencies: - sqlalchemy=2.0.23=py311h459d7ec_0 - stack_data=0.6.2=pyhd8ed1ab_0 - starlette=0.33.0=pyhd8ed1ab_0 - - tiledb=2.16.3=h8c794c1_3 + - tiledb=2.18.2=h8c794c1_0 - ukkonen=1.0.1=py311h9547e67_4 - uvicorn=0.24.0.post1=py311h38be061_0 - watchfiles=0.21.0=py311h46250e7_0 @@ -468,7 +468,7 @@ dependencies: - isoduration=20.11.0=pyhd8ed1ab_0 - jsonschema=4.20.0=pyhd8ed1ab_0 - jupyter_client=8.6.0=pyhd8ed1ab_0 - - libgdal=3.8.0=he7dcfe9_6 + - libgdal=3.8.1=hd0089ee_2 - numba=0.58.1=py311h96b013e_0 - numexpr=2.8.7=py311h039bad6_104 - oauthlib=3.2.2=pyhd8ed1ab_0 @@ -489,16 +489,16 @@ dependencies: - typer=0.9.0=pyhd8ed1ab_0 - uvicorn-standard=0.24.0.post1=h38be061_0 - aws-sdk-cpp=1.11.182=h8beafcf_7 - - boto3=1.33.10=pyhd8ed1ab_0 + - boto3=1.33.11=pyhd8ed1ab_0 - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0 - - dagster=1.5.10=pyhd8ed1ab_0 + - dagster=1.5.11=pyhd8ed1ab_0 - datasette=0.64.4=pyhd8ed1ab_1 - doc8=1.1.1=pyhd8ed1ab_0 - email-validator=2.1.0.post1=pyhd8ed1ab_0 - frictionless=4.40.8=pyh6c4a22f_0 - - gdal=3.8.0=py311h815a124_6 + - gdal=3.8.1=py311h67923c1_2 - geopandas-base=0.14.1=pyha770c72_0 - - google-auth=2.25.1=pyhca7485f_0 + - google-auth=2.25.2=pyhca7485f_0 - gql-with-requests=3.4.1=pyhd8ed1ab_0 - gtk2=2.24.33=h90689f9_2 - ipykernel=6.26.0=pyhf8b6a83_0 @@ -509,15 +509,15 @@ dependencies: - matplotlib-base=3.8.2=py311h54ef318_0 - nbformat=5.9.2=pyhd8ed1ab_0 - pandera-core=0.17.2=pyhd8ed1ab_1 - - pre-commit=3.5.0=pyha770c72_0 + - pre-commit=3.6.0=pyha770c72_0 - pydantic-settings=2.1.0=pyhd8ed1ab_1 - requests-oauthlib=1.3.1=pyhd8ed1ab_0 - scikit-learn=1.3.2=py311hc009520_2 - timezonefinder=6.2.0=py311h459d7ec_2 - catalystcoop.ferc_xbrl_extractor=1.3.1=pyhd8ed1ab_0 - conda-lock=2.5.1=pyhd8ed1ab_0 - - dagster-graphql=1.5.10=pyhd8ed1ab_0 - - dagster-postgres=0.21.10=pyhd8ed1ab_0 + - dagster-graphql=1.5.11=pyhd8ed1ab_0 + - dagster-postgres=0.21.11=pyhd8ed1ab_0 - fiona=1.9.5=py311hf8e0aa6_2 - google-api-core=2.15.0=pyhd8ed1ab_0 - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0 @@ -530,7 +530,7 @@ dependencies: - qtconsole-base=5.5.1=pyha770c72_0 - recordlinkage=0.16=pyhd8ed1ab_0 - tabulator=1.53.5=pyhd8ed1ab_0 - - dagster-webserver=1.5.10=pyhd8ed1ab_0 + - dagster-webserver=1.5.11=pyhd8ed1ab_0 - geopandas=0.14.1=pyhd8ed1ab_0 - google-cloud-core=2.4.1=pyhd8ed1ab_0 - libarrow-acero=14.0.1=h59595ed_3_cpu diff --git a/environments/conda-osx-64.lock.yml b/environments/conda-osx-64.lock.yml index 547833bf29..d9a3d57972 100644 --- a/environments/conda-osx-64.lock.yml +++ b/environments/conda-osx-64.lock.yml @@ -22,10 +22,10 @@ dependencies: - libbrotlicommon=1.1.0=h0dc2134_1 - libcxx=16.0.6=hd57cbcb_0 - libdeflate=1.19=ha4e1b8e_0 - - libev=4.33=haf1e3a3_1 + - libev=4.33=h10d778d_2 - libexpat=2.5.0=hf0c8a7f_1 - libffi=3.4.2=h0d85af4_5 - - libiconv=1.17=hac89ed1_0 + - libiconv=1.17=hd75f5a5_1 - libjpeg-turbo=3.0.0=h0dc2134_1 - libsodium=1.0.18=hbcb3906_1 - libutf8proc=2.8.0=hb7f2c08_0 @@ -70,7 +70,7 @@ dependencies: - lz4-c=1.9.4=hf0c8a7f_0 - ncurses=6.4=h93d8f39_2 - nspr=4.35=hea0b92c_0 - - openssl=3.1.4=hd75f5a5_0 + - openssl=3.2.0=hd75f5a5_1 - pandoc=3.1.3=h9d075a6_0 - pcre2=10.42=h0ad2156_0 - pixman=0.42.2=he965462_0 @@ -93,7 +93,7 @@ dependencies: - libglib=2.78.3=h198397b_0 - libkml=1.3.0=hab3ca0e_1018 - libllvm15=15.0.7=he4b1e75_3 - - libnghttp2=1.58.0=h64cf6d3_0 + - libnghttp2=1.58.0=h64cf6d3_1 - libprotobuf=4.24.4=h0ee05dc_0 - libre2-11=2023.06.02=h4694dbf_0 - librttopo=1.1.0=hf05f67e_15 @@ -120,7 +120,7 @@ dependencies: - libwebp=1.3.2=h44782d1_1 - openjpeg=2.5.0=ha4da562_3 - orc=1.9.0=hd1092d7_4 - - prettier=3.1.0=hbd11d21_0 + - prettier=3.1.1=hbd11d21_0 - python=3.11.6=h30d4d87_0_cpython - re2=2023.06.02=hd34609a_0 - sqlite=3.44.2=h7461747_0 @@ -152,7 +152,7 @@ dependencies: - colorama=0.4.6=pyhd8ed1ab_0 - crashtest=0.4.1=pyhd8ed1ab_0 - cycler=0.12.1=pyhd8ed1ab_0 - - dagster-pipes=1.5.10=pyhd8ed1ab_0 + - dagster-pipes=1.5.11=pyhd8ed1ab_0 - dataclasses=0.8=pyhc8e2a94_3 - debugpy=1.8.0=py311hdf8f085_1 - decorator=5.1.1=pyhd8ed1ab_0 @@ -169,7 +169,7 @@ dependencies: - frozenlist=1.4.0=py311h2725bcf_1 - fsspec=2023.12.1=pyhca7485f_0 - google-cloud-sdk=456.0.0=py311h6eed73b_0 - - greenlet=3.0.1=py311hd39e593_0 + - greenlet=3.0.2=py311hd39e593_0 - hpack=4.0.0=pyh9f0ad1d_0 - httptools=0.6.1=py311he705e18_0 - humanfriendly=10.0=pyhd8ed1ab_6 @@ -187,10 +187,10 @@ dependencies: - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0 - kiwisolver=1.4.5=py311h5fe6e05_1 - libblas=3.9.0=20_osx64_openblas - - libcurl=8.4.0=h726d00d_0 + - libcurl=8.5.0=h726d00d_0 - libgd=2.3.3=h0dceb68_9 - libgrpc=1.59.2=ha7f534c_0 - - libpq=16.1=h6dd4ff7_0 + - libpq=16.1=ha925e61_7 - llvmlite=0.41.1=py311hb5c2e0a_0 - locket=1.0.0=pyhd8ed1ab_0 - lxml=4.9.3=py311h719c1e2_2 @@ -270,7 +270,7 @@ dependencies: - traitlets=5.14.0=pyhd8ed1ab_0 - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0 - types-pyyaml=6.0.12.12=pyhd8ed1ab_0 - - typing_extensions=4.8.0=pyha770c72_0 + - typing_extensions=4.9.0=pyha770c72_0 - typing_utils=0.1.0=pyhd8ed1ab_0 - unicodecsv=0.14.1=pyhd8ed1ab_2 - uri-template=1.3.0=pyhd8ed1ab_0 @@ -300,7 +300,7 @@ dependencies: - bleach=6.1.0=pyhd8ed1ab_0 - cached-property=1.5.2=hd8ed1ab_1 - cffi=1.16.0=py311hc0b63fd_0 - - cfitsio=4.3.0=h66f91ea_0 + - cfitsio=4.3.1=h60fb419_0 - click-default-group=1.2.4=pyhd8ed1ab_0 - click-default-group-wheel=1.2.2=pyhd8ed1ab_0 - click-plugins=1.1.1=py_0 @@ -309,7 +309,7 @@ dependencies: - coloredlogs=14.0=pyhd8ed1ab_3 - comm=0.1.4=pyhd8ed1ab_0 - coverage=7.3.2=py311h2725bcf_0 - - curl=8.4.0=h726d00d_0 + - curl=8.5.0=h726d00d_0 - fonttools=4.46.0=py311he705e18_0 - gitdb=4.0.11=pyhd8ed1ab_0 - graphql-core=3.2.3=pyhd8ed1ab_0 @@ -317,9 +317,9 @@ dependencies: - h11=0.14.0=pyhd8ed1ab_0 - h2=4.1.0=pyhd8ed1ab_0 - harfbuzz=8.3.0=hf45c392_0 - - hdf5=1.14.2=nompi_hedada53_100 + - hdf5=1.14.3=nompi_h691f4bf_100 - html5lib=1.1=pyh9f0ad1d_0 - - hypothesis=6.91.1=pyha770c72_0 + - hypothesis=6.92.0=pyha770c72_0 - importlib-metadata=7.0.0=pyha770c72_0 - importlib_resources=6.1.1=pyhd8ed1ab_0 - isodate=0.6.1=pyhd8ed1ab_0 @@ -343,16 +343,16 @@ dependencies: - overrides=7.4.0=pyhd8ed1ab_0 - partd=1.4.1=pyhd8ed1ab_0 - pexpect=4.8.0=pyh1a96a4e_2 - - pint=0.22=pyhd8ed1ab_1 + - pint=0.23=pyhd8ed1ab_0 - pip=23.3.1=pyhd8ed1ab_0 - - poppler=23.11.0=hdd5a5e8_0 - - postgresql=16.1=h413614c_0 + - poppler=23.12.0=hdd5a5e8_0 + - postgresql=16.1=h413614c_7 - proj=9.3.0=h23b96cc_2 - prompt-toolkit=3.0.41=pyha770c72_0 - protobuf=4.24.4=py311h021eaf5_0 - - psycopg2=2.9.7=py311h187f0af_1 + - psycopg2=2.9.9=py311h187f0af_0 - pyasn1-modules=0.3.0=pyhd8ed1ab_0 - - pyobjc-core=10.0=py311hf110eff_0 + - pyobjc-core=10.1=py311h9b70068_0 - pyproject_hooks=1.0.0=pyhd8ed1ab_0 - pytest=7.4.3=pyhd8ed1ab_0 - python-dateutil=2.8.2=pyhd8ed1ab_0 @@ -367,7 +367,7 @@ dependencies: - terminado=0.18.0=pyh31c8845_0 - tinycss2=1.2.1=pyhd8ed1ab_0 - tqdm=4.66.1=pyhd8ed1ab_0 - - typing-extensions=4.8.0=hd8ed1ab_0 + - typing-extensions=4.9.0=hd8ed1ab_0 - typing_inspect=0.9.0=pyhd8ed1ab_0 - universal_pathlib=0.1.4=pyhd8ed1ab_0 - urllib3=1.26.18=pyhd8ed1ab_0 @@ -382,7 +382,7 @@ dependencies: - arrow=1.3.0=pyhd8ed1ab_0 - async-timeout=4.0.3=pyhd8ed1ab_0 - aws-crt-cpp=0.24.7=hf3941dc_6 - - botocore=1.33.10=pyhd8ed1ab_0 + - botocore=1.33.11=pyhd8ed1ab_0 - branca=0.7.0=pyhd8ed1ab_1 - croniter=2.0.1=pyhd8ed1ab_0 - cryptography=41.0.7=py311h48c7838_1 @@ -397,7 +397,7 @@ dependencies: - httpcore=1.0.2=pyhd8ed1ab_0 - importlib_metadata=7.0.0=hd8ed1ab_0 - jsonschema-specifications=2023.11.2=pyhd8ed1ab_0 - - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 + - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0 - kealib=1.5.2=h052fcf7_1 - libnetcdf=4.9.2=nompi_h6a32802_112 - libspatialite=5.1.0=hf63aa75_2 @@ -407,10 +407,10 @@ dependencies: - pbr=6.0.0=pyhd8ed1ab_0 - pendulum=2.1.2=py311h2725bcf_6 - prompt_toolkit=3.0.41=hd8ed1ab_0 - - psycopg2-binary=2.9.7=pyhd8ed1ab_1 + - psycopg2-binary=2.9.9=pyhd8ed1ab_0 - pybtex=0.24.0=pyhd8ed1ab_2 - pydantic-core=2.14.5=py311h5e0f0e4_0 - - pyobjc-framework-cocoa=10.0=py311hf110eff_1 + - pyobjc-framework-cocoa=10.1=py311h9b70068_0 - pyproj=3.6.1=py311he36daed_4 - pytest-console-scripts=1.4.1=pyhd8ed1ab_0 - pytest-cov=4.1.0=pyhd8ed1ab_0 @@ -422,7 +422,7 @@ dependencies: - sqlalchemy=2.0.23=py311he705e18_0 - stack_data=0.6.2=pyhd8ed1ab_0 - starlette=0.33.0=pyhd8ed1ab_0 - - tiledb=2.16.3=hd3a41d5_3 + - tiledb=2.18.2=h9fe0a6a_0 - ukkonen=1.0.1=py311h5fe6e05_4 - uvicorn=0.24.0.post1=py311h6eed73b_0 - watchfiles=0.21.0=py311h5e0f0e4_0 @@ -450,7 +450,7 @@ dependencies: - jsonschema=4.20.0=pyhd8ed1ab_0 - jupyter_client=8.6.0=pyhd8ed1ab_0 - keyring=24.3.0=py311h6eed73b_0 - - libgdal=3.8.0=h5b0c7d5_6 + - libgdal=3.8.1=h7bb8e92_2 - librsvg=2.56.3=hec3db73_0 - numba=0.58.1=py311h32f2313_0 - numexpr=2.8.7=py311h1eadf79_4 @@ -470,16 +470,16 @@ dependencies: - typeguard=4.1.5=pyhd8ed1ab_1 - typer=0.9.0=pyhd8ed1ab_0 - uvicorn-standard=0.24.0.post1=h6eed73b_0 - - boto3=1.33.10=pyhd8ed1ab_0 + - boto3=1.33.11=pyhd8ed1ab_0 - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0 - - dagster=1.5.10=pyhd8ed1ab_0 + - dagster=1.5.11=pyhd8ed1ab_0 - datasette=0.64.4=pyhd8ed1ab_1 - doc8=1.1.1=pyhd8ed1ab_0 - email-validator=2.1.0.post1=pyhd8ed1ab_0 - frictionless=4.40.8=pyh6c4a22f_0 - - gdal=3.8.0=py311h5646c56_6 + - gdal=3.8.1=py311h06f03dd_2 - geopandas-base=0.14.1=pyha770c72_0 - - google-auth=2.25.1=pyhca7485f_0 + - google-auth=2.25.2=pyhca7485f_0 - gql-with-requests=3.4.1=pyhd8ed1ab_0 - graphviz=9.0.0=hee74176_1 - ipykernel=6.26.0=pyh3cd1d5f_0 @@ -489,15 +489,15 @@ dependencies: - matplotlib-base=3.8.2=py311hd316c10_0 - nbformat=5.9.2=pyhd8ed1ab_0 - pandera-core=0.17.2=pyhd8ed1ab_1 - - pre-commit=3.5.0=pyha770c72_0 + - pre-commit=3.6.0=pyha770c72_0 - pydantic-settings=2.1.0=pyhd8ed1ab_1 - requests-oauthlib=1.3.1=pyhd8ed1ab_0 - scikit-learn=1.3.2=py311h66081b9_2 - timezonefinder=6.2.0=py311he705e18_2 - catalystcoop.ferc_xbrl_extractor=1.3.1=pyhd8ed1ab_0 - conda-lock=2.5.1=pyhd8ed1ab_0 - - dagster-graphql=1.5.10=pyhd8ed1ab_0 - - dagster-postgres=0.21.10=pyhd8ed1ab_0 + - dagster-graphql=1.5.11=pyhd8ed1ab_0 + - dagster-postgres=0.21.11=pyhd8ed1ab_0 - fiona=1.9.5=py311h809632c_2 - google-api-core=2.15.0=pyhd8ed1ab_0 - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0 @@ -513,7 +513,7 @@ dependencies: - qtconsole-base=5.5.1=pyha770c72_0 - recordlinkage=0.16=pyhd8ed1ab_0 - tabulator=1.53.5=pyhd8ed1ab_0 - - dagster-webserver=1.5.10=pyhd8ed1ab_0 + - dagster-webserver=1.5.11=pyhd8ed1ab_0 - geopandas=0.14.1=pyhd8ed1ab_0 - google-cloud-core=2.4.1=pyhd8ed1ab_0 - libarrow-dataset=14.0.1=hc222712_3_cpu diff --git a/environments/conda-osx-arm64.lock.yml b/environments/conda-osx-arm64.lock.yml index 6234f14f3b..8637bccd2c 100644 --- a/environments/conda-osx-arm64.lock.yml +++ b/environments/conda-osx-arm64.lock.yml @@ -22,10 +22,10 @@ dependencies: - libbrotlicommon=1.1.0=hb547adb_1 - libcxx=16.0.6=h4653b0c_0 - libdeflate=1.19=hb547adb_0 - - libev=4.33=h642e427_1 + - libev=4.33=h93a5062_2 - libexpat=2.5.0=hb7217d7_1 - libffi=3.4.2=h3422bc3_5 - - libiconv=1.17=he4db4b2_0 + - libiconv=1.17=h0d3ecfb_1 - libjpeg-turbo=3.0.0=hb547adb_1 - libsodium=1.0.18=h27ca646_1 - libutf8proc=2.8.0=h1a8c8d9_0 @@ -71,7 +71,7 @@ dependencies: - lz4-c=1.9.4=hb7217d7_0 - ncurses=6.4=h463b476_2 - nspr=4.35=hb7217d7_0 - - openssl=3.1.4=h0d3ecfb_0 + - openssl=3.2.0=h0d3ecfb_1 - pcre2=10.42=h26f9a81_0 - pixman=0.42.2=h13dd4ca_0 - snappy=1.1.10=h17c5cce_0 @@ -93,7 +93,7 @@ dependencies: - libglib=2.78.3=hb438215_0 - libkml=1.3.0=h1eb4d9f_1018 - libllvm15=15.0.7=h504e6bf_3 - - libnghttp2=1.58.0=ha4dd798_0 + - libnghttp2=1.58.0=ha4dd798_1 - libprotobuf=4.24.4=hc9861d8_0 - libre2-11=2023.06.02=h1753957_0 - librttopo=1.1.0=hc8f776e_15 @@ -120,7 +120,7 @@ dependencies: - libwebp=1.3.2=hf30222e_1 - openjpeg=2.5.0=h4c1507b_3 - orc=1.9.0=h7c018df_4 - - prettier=3.1.0=hb67532b_0 + - prettier=3.1.1=hb67532b_0 - python=3.11.6=h47c9636_0_cpython - re2=2023.06.02=h6135d0a_0 - sqlite=3.44.2=hf2abe2d_0 @@ -152,7 +152,7 @@ dependencies: - colorama=0.4.6=pyhd8ed1ab_0 - crashtest=0.4.1=pyhd8ed1ab_0 - cycler=0.12.1=pyhd8ed1ab_0 - - dagster-pipes=1.5.10=pyhd8ed1ab_0 + - dagster-pipes=1.5.11=pyhd8ed1ab_0 - dataclasses=0.8=pyhc8e2a94_3 - debugpy=1.8.0=py311ha891d26_1 - decorator=5.1.1=pyhd8ed1ab_0 @@ -169,7 +169,7 @@ dependencies: - frozenlist=1.4.0=py311heffc1b2_1 - fsspec=2023.12.1=pyhca7485f_0 - google-cloud-sdk=456.0.0=py311h267d04e_0 - - greenlet=3.0.1=py311hbaf5611_0 + - greenlet=3.0.2=py311hbaf5611_0 - hpack=4.0.0=pyh9f0ad1d_0 - httptools=0.6.1=py311h05b510d_0 - humanfriendly=10.0=pyhd8ed1ab_6 @@ -187,10 +187,10 @@ dependencies: - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0 - kiwisolver=1.4.5=py311he4fd1f5_1 - libblas=3.9.0=20_osxarm64_openblas - - libcurl=8.4.0=h2d989ff_0 + - libcurl=8.5.0=h2d989ff_0 - libgd=2.3.3=hfdf3952_9 - libgrpc=1.59.2=hbcf6334_0 - - libpq=16.1=hd435d45_0 + - libpq=16.1=h0f8b458_7 - llvmlite=0.41.1=py311hf5d242d_0 - locket=1.0.0=pyhd8ed1ab_0 - lxml=4.9.3=py311hdef8331_2 @@ -270,7 +270,7 @@ dependencies: - traitlets=5.14.0=pyhd8ed1ab_0 - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0 - types-pyyaml=6.0.12.12=pyhd8ed1ab_0 - - typing_extensions=4.8.0=pyha770c72_0 + - typing_extensions=4.9.0=pyha770c72_0 - typing_utils=0.1.0=pyhd8ed1ab_0 - unicodecsv=0.14.1=pyhd8ed1ab_2 - uri-template=1.3.0=pyhd8ed1ab_0 @@ -300,7 +300,7 @@ dependencies: - bleach=6.1.0=pyhd8ed1ab_0 - cached-property=1.5.2=hd8ed1ab_1 - cffi=1.16.0=py311h4a08483_0 - - cfitsio=4.3.0=hca87796_0 + - cfitsio=4.3.1=h808cd33_0 - click-default-group=1.2.4=pyhd8ed1ab_0 - click-default-group-wheel=1.2.2=pyhd8ed1ab_0 - click-plugins=1.1.1=py_0 @@ -309,7 +309,7 @@ dependencies: - coloredlogs=14.0=pyhd8ed1ab_3 - comm=0.1.4=pyhd8ed1ab_0 - coverage=7.3.2=py311heffc1b2_0 - - curl=8.4.0=h2d989ff_0 + - curl=8.5.0=h2d989ff_0 - fonttools=4.46.0=py311h05b510d_0 - gitdb=4.0.11=pyhd8ed1ab_0 - graphql-core=3.2.3=pyhd8ed1ab_0 @@ -317,9 +317,9 @@ dependencies: - h11=0.14.0=pyhd8ed1ab_0 - h2=4.1.0=pyhd8ed1ab_0 - harfbuzz=8.3.0=h8f0ba13_0 - - hdf5=1.14.2=nompi_h3aba7b3_100 + - hdf5=1.14.3=nompi_h5bb55e9_100 - html5lib=1.1=pyh9f0ad1d_0 - - hypothesis=6.91.1=pyha770c72_0 + - hypothesis=6.92.0=pyha770c72_0 - importlib-metadata=7.0.0=pyha770c72_0 - importlib_resources=6.1.1=pyhd8ed1ab_0 - isodate=0.6.1=pyhd8ed1ab_0 @@ -343,16 +343,16 @@ dependencies: - overrides=7.4.0=pyhd8ed1ab_0 - partd=1.4.1=pyhd8ed1ab_0 - pexpect=4.8.0=pyh1a96a4e_2 - - pint=0.22=pyhd8ed1ab_1 + - pint=0.23=pyhd8ed1ab_0 - pip=23.3.1=pyhd8ed1ab_0 - - poppler=23.11.0=hcdd998b_0 - - postgresql=16.1=hc6ab77f_0 + - poppler=23.12.0=hcdd998b_0 + - postgresql=16.1=hc6ab77f_7 - proj=9.3.0=h52fb9d0_2 - prompt-toolkit=3.0.41=pyha770c72_0 - protobuf=4.24.4=py311h4d1eceb_0 - - psycopg2=2.9.7=py311h589e011_1 + - psycopg2=2.9.9=py311h589e011_0 - pyasn1-modules=0.3.0=pyhd8ed1ab_0 - - pyobjc-core=10.0=py311hb702dc4_0 + - pyobjc-core=10.1=py311h665608e_0 - pyproject_hooks=1.0.0=pyhd8ed1ab_0 - pytest=7.4.3=pyhd8ed1ab_0 - python-dateutil=2.8.2=pyhd8ed1ab_0 @@ -367,7 +367,7 @@ dependencies: - terminado=0.18.0=pyh31c8845_0 - tinycss2=1.2.1=pyhd8ed1ab_0 - tqdm=4.66.1=pyhd8ed1ab_0 - - typing-extensions=4.8.0=hd8ed1ab_0 + - typing-extensions=4.9.0=hd8ed1ab_0 - typing_inspect=0.9.0=pyhd8ed1ab_0 - universal_pathlib=0.1.4=pyhd8ed1ab_0 - urllib3=1.26.18=pyhd8ed1ab_0 @@ -382,7 +382,7 @@ dependencies: - arrow=1.3.0=pyhd8ed1ab_0 - async-timeout=4.0.3=pyhd8ed1ab_0 - aws-crt-cpp=0.24.7=hba4ac3b_6 - - botocore=1.33.10=pyhd8ed1ab_0 + - botocore=1.33.11=pyhd8ed1ab_0 - branca=0.7.0=pyhd8ed1ab_1 - croniter=2.0.1=pyhd8ed1ab_0 - cryptography=41.0.7=py311h08c85a6_1 @@ -397,7 +397,7 @@ dependencies: - httpcore=1.0.2=pyhd8ed1ab_0 - importlib_metadata=7.0.0=hd8ed1ab_0 - jsonschema-specifications=2023.11.2=pyhd8ed1ab_0 - - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1 + - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0 - kealib=1.5.2=h47b5e36_1 - libnetcdf=4.9.2=nompi_hb2fb864_112 - libspatialite=5.1.0=h66af7d6_2 @@ -407,10 +407,10 @@ dependencies: - pbr=6.0.0=pyhd8ed1ab_0 - pendulum=2.1.2=py311heffc1b2_6 - prompt_toolkit=3.0.41=hd8ed1ab_0 - - psycopg2-binary=2.9.7=pyhd8ed1ab_1 + - psycopg2-binary=2.9.9=pyhd8ed1ab_0 - pybtex=0.24.0=pyhd8ed1ab_2 - pydantic-core=2.14.5=py311h94f323b_0 - - pyobjc-framework-cocoa=10.0=py311hb702dc4_1 + - pyobjc-framework-cocoa=10.1=py311h665608e_0 - pyproj=3.6.1=py311h20a9b75_4 - pytest-console-scripts=1.4.1=pyhd8ed1ab_0 - pytest-cov=4.1.0=pyhd8ed1ab_0 @@ -422,7 +422,7 @@ dependencies: - sqlalchemy=2.0.23=py311h05b510d_0 - stack_data=0.6.2=pyhd8ed1ab_0 - starlette=0.33.0=pyhd8ed1ab_0 - - tiledb=2.16.3=he15c4da_3 + - tiledb=2.18.2=h555b8a3_0 - ukkonen=1.0.1=py311he4fd1f5_4 - uvicorn=0.24.0.post1=py311h267d04e_0 - watchfiles=0.21.0=py311h94f323b_0 @@ -450,12 +450,12 @@ dependencies: - jsonschema=4.20.0=pyhd8ed1ab_0 - jupyter_client=8.6.0=pyhd8ed1ab_0 - keyring=24.3.0=py311h267d04e_0 - - libgdal=3.8.0=h76f3012_6 + - libgdal=3.8.1=h1ac08d9_2 - librsvg=2.56.3=h0db3404_0 - numba=0.58.1=py311h9ec4793_0 - numexpr=2.8.7=py311h6e08293_4 - oauthlib=3.2.2=pyhd8ed1ab_0 - - pandas=2.1.3=py311h6e08293_0 + - pandas=2.1.4=py311h6e08293_0 - pybtex-docutils=1.0.3=py311h267d04e_1 - pydantic=2.5.2=pyhd8ed1ab_0 - pyopenssl=23.3.0=pyhd8ed1ab_0 @@ -470,16 +470,16 @@ dependencies: - typeguard=4.1.5=pyhd8ed1ab_1 - typer=0.9.0=pyhd8ed1ab_0 - uvicorn-standard=0.24.0.post1=ha1ab1f8_0 - - boto3=1.33.10=pyhd8ed1ab_0 + - boto3=1.33.11=pyhd8ed1ab_0 - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0 - - dagster=1.5.10=pyhd8ed1ab_0 + - dagster=1.5.11=pyhd8ed1ab_0 - datasette=0.64.4=pyhd8ed1ab_1 - doc8=1.1.1=pyhd8ed1ab_0 - email-validator=2.1.0.post1=pyhd8ed1ab_0 - frictionless=4.40.8=pyh6c4a22f_0 - - gdal=3.8.0=py311h32a4f3d_6 + - gdal=3.8.1=py311hff5d751_2 - geopandas-base=0.14.1=pyha770c72_0 - - google-auth=2.25.1=pyhca7485f_0 + - google-auth=2.25.2=pyhca7485f_0 - gql-with-requests=3.4.1=pyhd8ed1ab_0 - graphviz=9.0.0=h3face73_1 - ipykernel=6.26.0=pyh3cd1d5f_0 @@ -489,15 +489,15 @@ dependencies: - matplotlib-base=3.8.2=py311hfdba5f6_0 - nbformat=5.9.2=pyhd8ed1ab_0 - pandera-core=0.17.2=pyhd8ed1ab_1 - - pre-commit=3.5.0=pyha770c72_0 + - pre-commit=3.6.0=pyha770c72_0 - pydantic-settings=2.1.0=pyhd8ed1ab_1 - requests-oauthlib=1.3.1=pyhd8ed1ab_0 - scikit-learn=1.3.2=py311ha25ca4d_2 - timezonefinder=6.2.0=py311h05b510d_2 - catalystcoop.ferc_xbrl_extractor=1.3.1=pyhd8ed1ab_0 - conda-lock=2.5.1=pyhd8ed1ab_0 - - dagster-graphql=1.5.10=pyhd8ed1ab_0 - - dagster-postgres=0.21.10=pyhd8ed1ab_0 + - dagster-graphql=1.5.11=pyhd8ed1ab_0 + - dagster-postgres=0.21.11=pyhd8ed1ab_0 - fiona=1.9.5=py311h4760b73_2 - google-api-core=2.15.0=pyhd8ed1ab_0 - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0 @@ -513,7 +513,7 @@ dependencies: - qtconsole-base=5.5.1=pyha770c72_0 - recordlinkage=0.16=pyhd8ed1ab_0 - tabulator=1.53.5=pyhd8ed1ab_0 - - dagster-webserver=1.5.10=pyhd8ed1ab_0 + - dagster-webserver=1.5.11=pyhd8ed1ab_0 - geopandas=0.14.1=pyhd8ed1ab_0 - google-cloud-core=2.4.1=pyhd8ed1ab_0 - libarrow-dataset=14.0.1=had9dd58_3_cpu diff --git a/src/pudl/extract/ferc714.py b/src/pudl/extract/ferc714.py index 9e6c8ac6d7..b442bec6cb 100644 --- a/src/pudl/extract/ferc714.py +++ b/src/pudl/extract/ferc714.py @@ -2,7 +2,7 @@ from collections import OrderedDict import pandas as pd -from dagster import AssetOut, Output, multi_asset +from dagster import AssetsDefinition, asset import pudl @@ -63,40 +63,40 @@ """Dictionary mapping PUDL tables to FERC-714 filenames and character encodings.""" -@multi_asset( - outs={"raw_ferc714__" + table_name: AssetOut() for table_name in FERC714_FILES}, - required_resource_keys={"datastore", "dataset_settings"}, -) -def extract_ferc714(context): - """Extract the raw FERC Form 714 dataframes from their original CSV files. +def generate_raw_ferc714_asset(table_name: str) -> AssetsDefinition: + """Generates an asset for building the raw FERC 714 dataframe.""" + assert table_name in FERC714_FILES - Args: - context: dagster keyword that provides access to resources and config. + @asset( + name=f"raw_ferc714__{table_name}", + required_resource_keys={"datastore", "dataset_settings"}, + ) + def _extract_raw_ferc714(context): + """Extract the raw FERC Form 714 dataframes from their original CSV files. - Returns: - A tuple of extracted FERC-714 dataframes. - """ - ds = context.resources.datastore - ferc714_settings = context.resources.dataset_settings.ferc714 - years = ", ".join(map(str, ferc714_settings.years)) + Args: + context: dagster keyword that provides access to resources and config. + """ + ds = context.resources.datastore + ferc714_settings = context.resources.dataset_settings.ferc714 + years = ", ".join(map(str, ferc714_settings.years)) - raw_dfs: OrderedDict[str, pd.DataFrame] = OrderedDict({}) - for table_name in FERC714_FILES: logger.info( f"Extracting {table_name} from CSV into pandas DataFrame (years: {years})." ) - with ds.get_zipfile_resource("ferc714", name="ferc714.zip").open( - FERC714_FILES[table_name]["name"] - ) as f: - raw_dfs[table_name] = pd.read_csv( - f, encoding=FERC714_FILES[table_name]["encoding"] - ) + df = pd.read_csv( + ds.get_zipfile_resource("ferc714", name="ferc714.zip").open( + FERC714_FILES[table_name]["name"] + ), + encoding=FERC714_FILES[table_name]["encoding"], + ) if table_name != "respondent_id": - raw_dfs[table_name] = raw_dfs[table_name].query( - "report_yr in @ferc714_settings.years" - ) + df = df.query("report_yr in @ferc714_settings.years") + return df - return ( - Output(output_name="raw_ferc714__" + table_name, value=df) - for table_name, df in raw_dfs.items() - ) + return _extract_raw_ferc714 + + +raw_ferc714_assets = [ + generate_raw_ferc714_asset(table_name) for table_name in FERC714_FILES +]