diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a9bbdb9..d6bb65c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,6 +4,10 @@ updates: directory: "/" schedule: interval: "monthly" + groups: + pip-dependencies: + patterns: + - "*" # Raise pull requests for version updates # to pip against the `develop` branch target-branch: "develop" @@ -11,4 +15,8 @@ updates: directory: "/" schedule: interval: "monthly" + groups: + gha-dependencies: + patterns: + - "*" target-branch: "develop" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d2b510..21d3eb1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,9 @@ ---- +ci: + autoupdate_schedule: "monthly" # Like dependabot + autoupdate_commit_msg: "chore: update pre-commit hooks" + autoupdate_branch: "develop" + autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -15,7 +20,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.4.2' + rev: 'v0.4.10' hooks: - id: ruff args: [ "--fix" ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ff1514..fa6b8be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] + +### Added +- [issue #127](https://github.com/nasa/batchee/issues/127): Group dependabot updates into fewer PRs +- [issue #129](https://github.com/nasa/batchee/issues/129): Add autoupdate schedule for pre-commit +### Changed +- [issue #128](https://github.com/nasa/batchee/issues/128): Increase continuous integration/unit test coverage +### Deprecated +### Removed +### Fixed + ## [1.1.0] ### Added diff --git a/batcher/harmony/service_adapter.py b/batcher/harmony/service_adapter.py index 8723d57..b35ac51 100644 --- a/batcher/harmony/service_adapter.py +++ b/batcher/harmony/service_adapter.py @@ -125,7 +125,7 @@ def process_catalog(self, catalog: pystac.Catalog) -> list[pystac.Catalog]: _get_output_date_range([item]), ) output_item.add_asset( - f"data_{idx}", + "data", Asset( _get_item_url(item), title=_get_item_url(item), diff --git a/batcher/tempo_filename_parser.py b/batcher/tempo_filename_parser.py index dc7a346..ceb7c8a 100644 --- a/batcher/tempo_filename_parser.py +++ b/batcher/tempo_filename_parser.py @@ -28,7 +28,6 @@ import logging import re from argparse import ArgumentParser -from pathlib import Path default_logger = logging.getLogger(__name__) @@ -99,7 +98,7 @@ def main() -> list[list[str]]: if args.verbose: logging.basicConfig(level=logging.DEBUG) - input_filenames = [str(Path(f).resolve()) for f in args.file_names] + input_filenames = args.file_names batch_indices = get_batch_indices(input_filenames) unique_category_indices: list[int] = sorted(set(batch_indices), key=batch_indices.index) diff --git a/poetry.lock b/poetry.lock index f3e3abb..3c27217 100644 --- a/poetry.lock +++ b/poetry.lock @@ -48,17 +48,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.34.95" +version = "1.34.131" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.95-py3-none-any.whl", hash = "sha256:e836b71d79671270fccac0a4d4c8ec239a6b82ea47c399b64675aa597d0ee63b"}, - {file = "boto3-1.34.95.tar.gz", hash = "sha256:decf52f8d5d8a1b10c9ff2a0e96ee207ed79e33d2e53fdf0880a5cbef70785e0"}, + {file = "boto3-1.34.131-py3-none-any.whl", hash = "sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b"}, + {file = "boto3-1.34.131.tar.gz", hash = "sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216"}, ] [package.dependencies] -botocore = ">=1.34.95,<1.35.0" +botocore = ">=1.34.131,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -67,13 +67,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.95" +version = "1.34.131" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.95-py3-none-any.whl", hash = "sha256:ead5823e0dd6751ece5498cb979fd9abf190e691c8833bcac6876fd6ca261fa7"}, - {file = "botocore-1.34.95.tar.gz", hash = "sha256:6bd76a2eadb42b91fa3528392e981ad5b4dfdee3968fa5b904278acf6cbf15ff"}, + {file = "botocore-1.34.131-py3-none-any.whl", hash = "sha256:13b011d7b206ce00727dcee26548fa3b550db9046d5a0e90ac25a6e6c8fde6ef"}, + {file = "botocore-1.34.131.tar.gz", hash = "sha256:502ddafe1d627fcf1e4c007c86454e5dd011dba7c58bd8e8a5368a79f3e387dc"}, ] [package.dependencies] @@ -82,17 +82,17 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} [package.extras] -crt = ["awscrt (==0.20.9)"] +crt = ["awscrt (==0.20.11)"] [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] [[package]] @@ -285,63 +285,63 @@ files = [ [[package]] name = "coverage" -version = "7.5.0" +version = "7.5.4" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:432949a32c3e3f820af808db1833d6d1631664d53dd3ce487aa25d574e18ad1c"}, - {file = "coverage-7.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2bd7065249703cbeb6d4ce679c734bef0ee69baa7bff9724361ada04a15b7e3b"}, - {file = "coverage-7.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbfe6389c5522b99768a93d89aca52ef92310a96b99782973b9d11e80511f932"}, - {file = "coverage-7.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39793731182c4be939b4be0cdecde074b833f6171313cf53481f869937129ed3"}, - {file = "coverage-7.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85a5dbe1ba1bf38d6c63b6d2c42132d45cbee6d9f0c51b52c59aa4afba057517"}, - {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:357754dcdfd811462a725e7501a9b4556388e8ecf66e79df6f4b988fa3d0b39a"}, - {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a81eb64feded34f40c8986869a2f764f0fe2db58c0530d3a4afbcde50f314880"}, - {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:51431d0abbed3a868e967f8257c5faf283d41ec882f58413cf295a389bb22e58"}, - {file = "coverage-7.5.0-cp310-cp310-win32.whl", hash = "sha256:f609ebcb0242d84b7adeee2b06c11a2ddaec5464d21888b2c8255f5fd6a98ae4"}, - {file = "coverage-7.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:6782cd6216fab5a83216cc39f13ebe30adfac2fa72688c5a4d8d180cd52e8f6a"}, - {file = "coverage-7.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e768d870801f68c74c2b669fc909839660180c366501d4cc4b87efd6b0eee375"}, - {file = "coverage-7.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:84921b10aeb2dd453247fd10de22907984eaf80901b578a5cf0bb1e279a587cb"}, - {file = "coverage-7.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:710c62b6e35a9a766b99b15cdc56d5aeda0914edae8bb467e9c355f75d14ee95"}, - {file = "coverage-7.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c379cdd3efc0658e652a14112d51a7668f6bfca7445c5a10dee7eabecabba19d"}, - {file = "coverage-7.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fea9d3ca80bcf17edb2c08a4704259dadac196fe5e9274067e7a20511fad1743"}, - {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:41327143c5b1d715f5f98a397608f90ab9ebba606ae4e6f3389c2145410c52b1"}, - {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:565b2e82d0968c977e0b0f7cbf25fd06d78d4856289abc79694c8edcce6eb2de"}, - {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cf3539007202ebfe03923128fedfdd245db5860a36810136ad95a564a2fdffff"}, - {file = "coverage-7.5.0-cp311-cp311-win32.whl", hash = "sha256:bf0b4b8d9caa8d64df838e0f8dcf68fb570c5733b726d1494b87f3da85db3a2d"}, - {file = "coverage-7.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c6384cc90e37cfb60435bbbe0488444e54b98700f727f16f64d8bfda0b84656"}, - {file = "coverage-7.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fed7a72d54bd52f4aeb6c6e951f363903bd7d70bc1cad64dd1f087980d309ab9"}, - {file = "coverage-7.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cbe6581fcff7c8e262eb574244f81f5faaea539e712a058e6707a9d272fe5b64"}, - {file = "coverage-7.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad97ec0da94b378e593ef532b980c15e377df9b9608c7c6da3506953182398af"}, - {file = "coverage-7.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd4bacd62aa2f1a1627352fe68885d6ee694bdaebb16038b6e680f2924a9b2cc"}, - {file = "coverage-7.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adf032b6c105881f9d77fa17d9eebe0ad1f9bfb2ad25777811f97c5362aa07f2"}, - {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ba01d9ba112b55bfa4b24808ec431197bb34f09f66f7cb4fd0258ff9d3711b1"}, - {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f0bfe42523893c188e9616d853c47685e1c575fe25f737adf473d0405dcfa7eb"}, - {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a9a7ef30a1b02547c1b23fa9a5564f03c9982fc71eb2ecb7f98c96d7a0db5cf2"}, - {file = "coverage-7.5.0-cp312-cp312-win32.whl", hash = "sha256:3c2b77f295edb9fcdb6a250f83e6481c679335ca7e6e4a955e4290350f2d22a4"}, - {file = "coverage-7.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:427e1e627b0963ac02d7c8730ca6d935df10280d230508c0ba059505e9233475"}, - {file = "coverage-7.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9dd88fce54abbdbf4c42fb1fea0e498973d07816f24c0e27a1ecaf91883ce69e"}, - {file = "coverage-7.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a898c11dca8f8c97b467138004a30133974aacd572818c383596f8d5b2eb04a9"}, - {file = "coverage-7.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07dfdd492d645eea1bd70fb1d6febdcf47db178b0d99161d8e4eed18e7f62fe7"}, - {file = "coverage-7.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3d117890b6eee85887b1eed41eefe2e598ad6e40523d9f94c4c4b213258e4a4"}, - {file = "coverage-7.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6afd2e84e7da40fe23ca588379f815fb6dbbb1b757c883935ed11647205111cb"}, - {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a9960dd1891b2ddf13a7fe45339cd59ecee3abb6b8326d8b932d0c5da208104f"}, - {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ced268e82af993d7801a9db2dbc1d2322e786c5dc76295d8e89473d46c6b84d4"}, - {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7c211f25777746d468d76f11719e64acb40eed410d81c26cefac641975beb88"}, - {file = "coverage-7.5.0-cp38-cp38-win32.whl", hash = "sha256:262fffc1f6c1a26125d5d573e1ec379285a3723363f3bd9c83923c9593a2ac25"}, - {file = "coverage-7.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:eed462b4541c540d63ab57b3fc69e7d8c84d5957668854ee4e408b50e92ce26a"}, - {file = "coverage-7.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d0194d654e360b3e6cc9b774e83235bae6b9b2cac3be09040880bb0e8a88f4a1"}, - {file = "coverage-7.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33c020d3322662e74bc507fb11488773a96894aa82a622c35a5a28673c0c26f5"}, - {file = "coverage-7.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbdf2cae14a06827bec50bd58e49249452d211d9caddd8bd80e35b53cb04631"}, - {file = "coverage-7.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3235d7c781232e525b0761730e052388a01548bd7f67d0067a253887c6e8df46"}, - {file = "coverage-7.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2de4e546f0ec4b2787d625e0b16b78e99c3e21bc1722b4977c0dddf11ca84e"}, - {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4d0e206259b73af35c4ec1319fd04003776e11e859936658cb6ceffdeba0f5be"}, - {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2055c4fb9a6ff624253d432aa471a37202cd8f458c033d6d989be4499aed037b"}, - {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:075299460948cd12722a970c7eae43d25d37989da682997687b34ae6b87c0ef0"}, - {file = "coverage-7.5.0-cp39-cp39-win32.whl", hash = "sha256:280132aada3bc2f0fac939a5771db4fbb84f245cb35b94fae4994d4c1f80dae7"}, - {file = "coverage-7.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:c58536f6892559e030e6924896a44098bc1290663ea12532c78cef71d0df8493"}, - {file = "coverage-7.5.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:2b57780b51084d5223eee7b59f0d4911c31c16ee5aa12737c7a02455829ff067"}, - {file = "coverage-7.5.0.tar.gz", hash = "sha256:cf62d17310f34084c59c01e027259076479128d11e4661bb6c9acb38c5e19bb8"}, + {file = "coverage-7.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6cfb5a4f556bb51aba274588200a46e4dd6b505fb1a5f8c5ae408222eb416f99"}, + {file = "coverage-7.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2174e7c23e0a454ffe12267a10732c273243b4f2d50d07544a91198f05c48f47"}, + {file = "coverage-7.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2214ee920787d85db1b6a0bd9da5f8503ccc8fcd5814d90796c2f2493a2f4d2e"}, + {file = "coverage-7.5.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1137f46adb28e3813dec8c01fefadcb8c614f33576f672962e323b5128d9a68d"}, + {file = "coverage-7.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b385d49609f8e9efc885790a5a0e89f2e3ae042cdf12958b6034cc442de428d3"}, + {file = "coverage-7.5.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b4a474f799456e0eb46d78ab07303286a84a3140e9700b9e154cfebc8f527016"}, + {file = "coverage-7.5.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5cd64adedf3be66f8ccee418473c2916492d53cbafbfcff851cbec5a8454b136"}, + {file = "coverage-7.5.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e564c2cf45d2f44a9da56f4e3a26b2236504a496eb4cb0ca7221cd4cc7a9aca9"}, + {file = "coverage-7.5.4-cp310-cp310-win32.whl", hash = "sha256:7076b4b3a5f6d2b5d7f1185fde25b1e54eb66e647a1dfef0e2c2bfaf9b4c88c8"}, + {file = "coverage-7.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:018a12985185038a5b2bcafab04ab833a9a0f2c59995b3cec07e10074c78635f"}, + {file = "coverage-7.5.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:db14f552ac38f10758ad14dd7b983dbab424e731588d300c7db25b6f89e335b5"}, + {file = "coverage-7.5.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3257fdd8e574805f27bb5342b77bc65578e98cbc004a92232106344053f319ba"}, + {file = "coverage-7.5.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a6612c99081d8d6134005b1354191e103ec9705d7ba2754e848211ac8cacc6b"}, + {file = "coverage-7.5.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d45d3cbd94159c468b9b8c5a556e3f6b81a8d1af2a92b77320e887c3e7a5d080"}, + {file = "coverage-7.5.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed550e7442f278af76d9d65af48069f1fb84c9f745ae249c1a183c1e9d1b025c"}, + {file = "coverage-7.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7a892be37ca35eb5019ec85402c3371b0f7cda5ab5056023a7f13da0961e60da"}, + {file = "coverage-7.5.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8192794d120167e2a64721d88dbd688584675e86e15d0569599257566dec9bf0"}, + {file = "coverage-7.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:820bc841faa502e727a48311948e0461132a9c8baa42f6b2b84a29ced24cc078"}, + {file = "coverage-7.5.4-cp311-cp311-win32.whl", hash = "sha256:6aae5cce399a0f065da65c7bb1e8abd5c7a3043da9dceb429ebe1b289bc07806"}, + {file = "coverage-7.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:d2e344d6adc8ef81c5a233d3a57b3c7d5181f40e79e05e1c143da143ccb6377d"}, + {file = "coverage-7.5.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:54317c2b806354cbb2dc7ac27e2b93f97096912cc16b18289c5d4e44fc663233"}, + {file = "coverage-7.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:042183de01f8b6d531e10c197f7f0315a61e8d805ab29c5f7b51a01d62782747"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6bb74ed465d5fb204b2ec41d79bcd28afccf817de721e8a807d5141c3426638"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3d45ff86efb129c599a3b287ae2e44c1e281ae0f9a9bad0edc202179bcc3a2e"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5013ed890dc917cef2c9f765c4c6a8ae9df983cd60dbb635df8ed9f4ebc9f555"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1014fbf665fef86cdfd6cb5b7371496ce35e4d2a00cda501cf9f5b9e6fced69f"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3684bc2ff328f935981847082ba4fdc950d58906a40eafa93510d1b54c08a66c"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:581ea96f92bf71a5ec0974001f900db495488434a6928a2ca7f01eee20c23805"}, + {file = "coverage-7.5.4-cp312-cp312-win32.whl", hash = "sha256:73ca8fbc5bc622e54627314c1a6f1dfdd8db69788f3443e752c215f29fa87a0b"}, + {file = "coverage-7.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:cef4649ec906ea7ea5e9e796e68b987f83fa9a718514fe147f538cfeda76d7a7"}, + {file = "coverage-7.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdd31315fc20868c194130de9ee6bfd99755cc9565edff98ecc12585b90be882"}, + {file = "coverage-7.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:02ff6e898197cc1e9fa375581382b72498eb2e6d5fc0b53f03e496cfee3fac6d"}, + {file = "coverage-7.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d05c16cf4b4c2fc880cb12ba4c9b526e9e5d5bb1d81313d4d732a5b9fe2b9d53"}, + {file = "coverage-7.5.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5986ee7ea0795a4095ac4d113cbb3448601efca7f158ec7f7087a6c705304e4"}, + {file = "coverage-7.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df54843b88901fdc2f598ac06737f03d71168fd1175728054c8f5a2739ac3e4"}, + {file = "coverage-7.5.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ab73b35e8d109bffbda9a3e91c64e29fe26e03e49addf5b43d85fc426dde11f9"}, + {file = "coverage-7.5.4-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:aea072a941b033813f5e4814541fc265a5c12ed9720daef11ca516aeacd3bd7f"}, + {file = "coverage-7.5.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:16852febd96acd953b0d55fc842ce2dac1710f26729b31c80b940b9afcd9896f"}, + {file = "coverage-7.5.4-cp38-cp38-win32.whl", hash = "sha256:8f894208794b164e6bd4bba61fc98bf6b06be4d390cf2daacfa6eca0a6d2bb4f"}, + {file = "coverage-7.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:e2afe743289273209c992075a5a4913e8d007d569a406ffed0bd080ea02b0633"}, + {file = "coverage-7.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b95c3a8cb0463ba9f77383d0fa8c9194cf91f64445a63fc26fb2327e1e1eb088"}, + {file = "coverage-7.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d7564cc09dd91b5a6001754a5b3c6ecc4aba6323baf33a12bd751036c998be4"}, + {file = "coverage-7.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44da56a2589b684813f86d07597fdf8a9c6ce77f58976727329272f5a01f99f7"}, + {file = "coverage-7.5.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e16f3d6b491c48c5ae726308e6ab1e18ee830b4cdd6913f2d7f77354b33f91c8"}, + {file = "coverage-7.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbc5958cb471e5a5af41b0ddaea96a37e74ed289535e8deca404811f6cb0bc3d"}, + {file = "coverage-7.5.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a04e990a2a41740b02d6182b498ee9796cf60eefe40cf859b016650147908029"}, + {file = "coverage-7.5.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ddbd2f9713a79e8e7242d7c51f1929611e991d855f414ca9996c20e44a895f7c"}, + {file = "coverage-7.5.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b1ccf5e728ccf83acd313c89f07c22d70d6c375a9c6f339233dcf792094bcbf7"}, + {file = "coverage-7.5.4-cp39-cp39-win32.whl", hash = "sha256:56b4eafa21c6c175b3ede004ca12c653a88b6f922494b023aeb1e836df953ace"}, + {file = "coverage-7.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:65e528e2e921ba8fd67d9055e6b9f9e34b21ebd6768ae1c1723f4ea6ace1234d"}, + {file = "coverage-7.5.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:79b356f3dd5b26f3ad23b35c75dbdaf1f9e2450b6bcefc6d0825ea0aa3f86ca5"}, + {file = "coverage-7.5.4.tar.gz", hash = "sha256:a44963520b069e12789d0faea4e9fdb1e410cdc4aab89d94f7f55cbb7fef0353"}, ] [package.dependencies] @@ -380,13 +380,13 @@ test = ["pytest (>=6)"] [[package]] name = "harmony-service-lib" -version = "1.0.26" +version = "1.0.27" description = "A library for Python-based Harmony services to parse incoming messages, fetch data, stage data, and call back to Harmony" optional = false python-versions = ">=3.8" files = [ - {file = "harmony_service_lib-1.0.26-py3-none-any.whl", hash = "sha256:fce65b5334f055c1d28036ca74df72c974d8229e69e108238ad599f3c3e7a8f4"}, - {file = "harmony_service_lib-1.0.26.tar.gz", hash = "sha256:a6a7342633859dbaba59ab595121ac06cced94d9b2d5aba44abc7ddbdd8d8ab0"}, + {file = "harmony_service_lib-1.0.27-py3-none-any.whl", hash = "sha256:edf3aaa4b6f18addbfc4b54c20746d16ffe365a55ebaab0f4df47296b5900af3"}, + {file = "harmony_service_lib-1.0.27.tar.gz", hash = "sha256:7992bbed205e02ad15dcc35526e7a827e6c3054a65d5eb5b59ddfec160950ca1"}, ] [package.dependencies] @@ -491,13 +491,13 @@ files = [ [[package]] name = "packaging" -version = "24.0" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] @@ -513,13 +513,13 @@ files = [ [[package]] name = "platformdirs" -version = "4.2.1" +version = "4.2.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, - {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] [package.extras] @@ -598,13 +598,13 @@ validation = ["jsonschema (==3.2.0)"] [[package]] name = "pytest" -version = "8.2.0" +version = "8.2.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"}, - {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"}, + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, ] [package.dependencies] @@ -663,13 +663,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.3" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -684,28 +684,28 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruff" -version = "0.4.2" +version = "0.4.10" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.4.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d14dc8953f8af7e003a485ef560bbefa5f8cc1ad994eebb5b12136049bbccc5"}, - {file = "ruff-0.4.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:24016ed18db3dc9786af103ff49c03bdf408ea253f3cb9e3638f39ac9cf2d483"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2e06459042ac841ed510196c350ba35a9b24a643e23db60d79b2db92af0c2b"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3afabaf7ba8e9c485a14ad8f4122feff6b2b93cc53cd4dad2fd24ae35112d5c5"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:799eb468ea6bc54b95527143a4ceaf970d5aa3613050c6cff54c85fda3fde480"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ec4ba9436a51527fb6931a8839af4c36a5481f8c19e8f5e42c2f7ad3a49f5069"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a2243f8f434e487c2a010c7252150b1fdf019035130f41b77626f5655c9ca22"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8772130a063f3eebdf7095da00c0b9898bd1774c43b336272c3e98667d4fb8fa"}, - {file = "ruff-0.4.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ab165ef5d72392b4ebb85a8b0fbd321f69832a632e07a74794c0e598e7a8376"}, - {file = "ruff-0.4.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1f32cadf44c2020e75e0c56c3408ed1d32c024766bd41aedef92aa3ca28eef68"}, - {file = "ruff-0.4.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:22e306bf15e09af45ca812bc42fa59b628646fa7c26072555f278994890bc7ac"}, - {file = "ruff-0.4.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:82986bb77ad83a1719c90b9528a9dd663c9206f7c0ab69282af8223566a0c34e"}, - {file = "ruff-0.4.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:652e4ba553e421a6dc2a6d4868bc3b3881311702633eb3672f9f244ded8908cd"}, - {file = "ruff-0.4.2-py3-none-win32.whl", hash = "sha256:7891ee376770ac094da3ad40c116258a381b86c7352552788377c6eb16d784fe"}, - {file = "ruff-0.4.2-py3-none-win_amd64.whl", hash = "sha256:5ec481661fb2fd88a5d6cf1f83403d388ec90f9daaa36e40e2c003de66751798"}, - {file = "ruff-0.4.2-py3-none-win_arm64.whl", hash = "sha256:cbd1e87c71bca14792948c4ccb51ee61c3296e164019d2d484f3eaa2d360dfaf"}, - {file = "ruff-0.4.2.tar.gz", hash = "sha256:33bcc160aee2520664bc0859cfeaebc84bb7323becff3f303b8f1f2d81cb4edc"}, + {file = "ruff-0.4.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c2c4d0859305ac5a16310eec40e4e9a9dec5dcdfbe92697acd99624e8638dac"}, + {file = "ruff-0.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a79489607d1495685cdd911a323a35871abfb7a95d4f98fc6f85e799227ac46e"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1dd1681dfa90a41b8376a61af05cc4dc5ff32c8f14f5fe20dba9ff5deb80cd6"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c75c53bb79d71310dc79fb69eb4902fba804a81f374bc86a9b117a8d077a1784"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18238c80ee3d9100d3535d8eb15a59c4a0753b45cc55f8bf38f38d6a597b9739"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:d8f71885bce242da344989cae08e263de29752f094233f932d4f5cfb4ef36a81"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:330421543bd3222cdfec481e8ff3460e8702ed1e58b494cf9d9e4bf90db52b9d"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e9b6fb3a37b772628415b00c4fc892f97954275394ed611056a4b8a2631365e"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f54c481b39a762d48f64d97351048e842861c6662d63ec599f67d515cb417f6"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:67fe086b433b965c22de0b4259ddfe6fa541c95bf418499bedb9ad5fb8d1c631"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:acfaaab59543382085f9eb51f8e87bac26bf96b164839955f244d07125a982ef"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3cea07079962b2941244191569cf3a05541477286f5cafea638cd3aa94b56815"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:338a64ef0748f8c3a80d7f05785930f7965d71ca260904a9321d13be24b79695"}, + {file = "ruff-0.4.10-py3-none-win32.whl", hash = "sha256:ffe3cd2f89cb54561c62e5fa20e8f182c0a444934bf430515a4b422f1ab7b7ca"}, + {file = "ruff-0.4.10-py3-none-win_amd64.whl", hash = "sha256:67f67cef43c55ffc8cc59e8e0b97e9e60b4837c8f21e8ab5ffd5d66e196e25f7"}, + {file = "ruff-0.4.10-py3-none-win_arm64.whl", hash = "sha256:dd1fcee327c20addac7916ca4e2653fbbf2e8388d8a6477ce5b4e986b68ae6c0"}, + {file = "ruff-0.4.10.tar.gz", hash = "sha256:3aa4f2bc388a30d346c56524f7cacca85945ba124945fe489952aadb6b5cd804"}, ] [[package]] @@ -749,24 +749,24 @@ files = [ [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] name = "urllib3" -version = "1.26.18" +version = "1.26.19" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, - {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, + {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"}, + {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"}, ] [package.extras] diff --git a/pyproject.toml b/pyproject.toml index b2c93ba..bfa7bff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "batchee" -version = "1.1.0" +version = "1.2.0rc1" description = "Determine how to group together input files into batches for subsequent concatenation" authors = ["Daniel Kaufman "] readme = "README.md" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bdfc664 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +from pathlib import Path + +import pytest + + +def pytest_addoption(parser): + """Sets up optional argument to keep temporary testing directory.""" + parser.addoption( + "--keep-tmp", + action="store_true", + help="Keep temporary directory after testing. Useful for debugging.", + ) + + +@pytest.fixture(scope="class") +def pass_options(request): + """Adds optional argument to a test class.""" + request.cls.KEEP_TMP = request.config.getoption("--keep-tmp") + + +@pytest.fixture(scope="function", autouse=True) +def temp_output_dir(tmpdir_factory) -> Path: + return Path(tmpdir_factory.mktemp("tmp-")) diff --git a/tests/data/harmony/message.json b/tests/data/harmony/message.json new file mode 100644 index 0000000..21eb885 --- /dev/null +++ b/tests/data/harmony/message.json @@ -0,0 +1,16 @@ +{ + "sources": [{ + "collection": "C1234088182-EEDTEST" + }], + "format": { + "mime": "application/x-netcdf4" + }, + "subset": {}, + "requestId": "00001111-2222-3333-4444-555566667777", + "user": "jdoe", + "client": "harmony-example", + "isSynchronous": false, + "stagingLocation": "s3://example-bucket/public/some-org/some-service/some-uuid/", + "callback": "http://localhost/some-path", + "version": "0.10.0" +} diff --git a/tests/data/harmony/source/catalog.json b/tests/data/harmony/source/catalog.json new file mode 100644 index 0000000..fd99944 --- /dev/null +++ b/tests/data/harmony/source/catalog.json @@ -0,0 +1,48 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59", + "links": [ + { + "rel": "harmony_source", + "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST" + }, + { + "rel": "item", + "href": "./granule_S012G01.json", + "type": "application/json", + "title": "granule_S012G01" + }, + { + "rel": "item", + "href": "./granule_S012G02.json", + "type": "application/json", + "title": "granule_S012G02" + }, + { + "rel": "item", + "href": "./granule_S013G01.json", + "type": "application/json", + "title": "granule_S013G01" + }, + { + "rel": "item", + "href": "./granule_S013G02.json", + "type": "application/json", + "title": "granule_S013G02" + }, + { + "rel": "item", + "href": "./granule_S014G01.json", + "type": "application/json", + "title": "granule_S014G01" + }, + { + "rel": "item", + "href": "./granule_S014G02.json", + "type": "application/json", + "title": "granule_S014G02" + } + ], + "description": "CMR Granules for C1234088182-EEDTEST batch 1" +} diff --git a/tests/data/harmony/source/catalog0.json b/tests/data/harmony/source/catalog0.json new file mode 100644 index 0000000..009421c --- /dev/null +++ b/tests/data/harmony/source/catalog0.json @@ -0,0 +1,36 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59", + "links": [ + { + "rel": "harmony_source", + "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST" + }, + { + "rel": "item", + "href": "./granule_S012G01.json", + "type": "application/json", + "title": "granule_S012G01" + }, + { + "rel": "item", + "href": "./granule_S012G02.json", + "type": "application/json", + "title": "granule_S012G02" + }, + { + "rel": "item", + "href": "./granule_S013G01.json", + "type": "application/json", + "title": "granule_S013G01" + }, + { + "rel": "next", + "href": "tests/data/harmony/source/catalog1.json", + "type": "application/json", + "title": "Next page" + } + ], + "description": "CMR Granules for C1234088182-EEDTEST batch 1" +} diff --git a/tests/data/harmony/source/catalog1.json b/tests/data/harmony/source/catalog1.json new file mode 100644 index 0000000..c844b16 --- /dev/null +++ b/tests/data/harmony/source/catalog1.json @@ -0,0 +1,36 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cfc32383-cfd1-4e43-8d5f-55f539b6fa59", + "links": [ + { + "rel": "harmony_source", + "href": "https://cmr.uat.earthdata.nasa.gov/search/concepts/C1234088182-EEDTEST" + }, + { + "rel": "item", + "href": "./granule_S013G02.json", + "type": "application/json", + "title": "granule_S013G02" + }, + { + "rel": "item", + "href": "./granule_S014G01.json", + "type": "application/json", + "title": "granule_S014G01" + }, + { + "rel": "item", + "href": "./granule_S014G02.json", + "type": "application/json", + "title": "granule_S014G02" + }, + { + "rel": "prev", + "href": "tests/data/harmony/source/catalog0.json", + "type": "application/json", + "title": "Previous page" + } + ], + "description": "CMR Granules for C1234088182-EEDTEST batch 2" +} diff --git a/tests/data/harmony/source/granule_S012G01.json b/tests/data/harmony/source/granule_S012G01.json new file mode 100644 index 0000000..f468f54 --- /dev/null +++ b/tests/data/harmony/source/granule_S012G01.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "51d02b24-d00e-4640-9887-05f98f6b96d8", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-02T00:00:00.000Z", + "end_datetime": "2020-01-02T23:59:59.000Z" + }, + "bbox": [1, 3, 1, 3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120101Z_S012G01.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120101Z_S012G01.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S012G02.json b/tests/data/harmony/source/granule_S012G02.json new file mode 100644 index 0000000..1d25a15 --- /dev/null +++ b/tests/data/harmony/source/granule_S012G02.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "21eb7dc5-7a9d-4374-9a88-12451ba653ef", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-03T00:00:00.000Z", + "end_datetime": "2020-01-03T23:59:59.000Z" + }, + "bbox": [-1, -3, -1, -3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120107Z_S012G02.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120107Z_S012G02.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S013G01.json b/tests/data/harmony/source/granule_S013G01.json new file mode 100644 index 0000000..c22cbc2 --- /dev/null +++ b/tests/data/harmony/source/granule_S013G01.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "dc07a25a-54bd-4389-97b6-c1d68cf0d586", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-05T00:00:00.000Z", + "end_datetime": "2020-01-05T23:59:59.000Z" + }, + "bbox": [-4, -2, -4, 2], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120202Z_S013G01.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120202Z_S013G01.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S013G02.json b/tests/data/harmony/source/granule_S013G02.json new file mode 100644 index 0000000..98d22f9 --- /dev/null +++ b/tests/data/harmony/source/granule_S013G02.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "53cdf09b-4004-4405-9363-f93455a11c2f", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-04T00:00:00.000Z", + "end_datetime": "2020-01-04T23:59:59.000Z" + }, + "bbox": [4, 2, 4, 2], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120209Z_S013G02.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120209Z_S013G02.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S014G01.json b/tests/data/harmony/source/granule_S014G01.json new file mode 100644 index 0000000..8800f8a --- /dev/null +++ b/tests/data/harmony/source/granule_S014G01.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "349b5f4a-34ab-4b0d-ae00-b55fe3d166fb", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-05T00:00:00.000Z", + "end_datetime": "2020-01-05T23:59:59.000Z" + }, + "bbox": [-5, -3, -5, 3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120303Z_S014G01.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120303Z_S014G01.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/data/harmony/source/granule_S014G02.json b/tests/data/harmony/source/granule_S014G02.json new file mode 100644 index 0000000..6f6dffc --- /dev/null +++ b/tests/data/harmony/source/granule_S014G02.json @@ -0,0 +1,49 @@ +{ + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "id": "cb0271ad-05a9-4138-8bfb-fa6b83fbe6fa", + "type": "Feature", + "links": [], + "properties": { + "start_datetime": "2020-01-05T00:00:00.000Z", + "end_datetime": "2020-01-05T23:59:59.000Z" + }, + "bbox": [5, 3, 5, 3], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -179.95, + -89.95 + ], + [ + -179.95, + 89.95 + ], + [ + 179.95, + 89.95 + ], + [ + 179.95, + -89.95 + ], + [ + -179.95, + -89.95 + ] + ] + ] + }, + "assets": { + "data": { + "href": "file://tests/data/harmony/granules/TEMPO_NO2_L2_V03_20240601T120310Z_S014G02.nc", + "title": "TEMPO_NO2_L2_V03_20240601T120310Z_S014G02.nc", + "type": "application/x-netcdf4", + "roles": [ + "data" + ] + } + } +} diff --git a/tests/test_filename_grouping.py b/tests/test_filename_grouping.py index 83ae365..368e168 100644 --- a/tests/test_filename_grouping.py +++ b/tests/test_filename_grouping.py @@ -1,3 +1,7 @@ +import sys +from unittest.mock import patch + +import batcher.tempo_filename_parser from batcher.tempo_filename_parser import get_batch_indices example_filenames = [ @@ -14,3 +18,13 @@ def test_grouping(): results = get_batch_indices(example_filenames) assert results == [0, 0, 0, 1, 1, 1] + + +def test_main_cli(): + test_args = [batcher.tempo_filename_parser.__file__, "-v"] + test_args.extend(example_filenames) + + with patch.object(sys, "argv", test_args): + grouped_names = batcher.tempo_filename_parser.main() + + assert grouped_names == [example_filenames[0:3], example_filenames[3:6]] diff --git a/tests/test_harmony_adapter.py b/tests/test_harmony_adapter.py new file mode 100644 index 0000000..2726a6e --- /dev/null +++ b/tests/test_harmony_adapter.py @@ -0,0 +1,101 @@ +import json +import sys +from os import environ +from pathlib import Path +from unittest.mock import patch +from urllib.parse import urlsplit + +import pytest + +import batcher.harmony.cli + + +@pytest.mark.usefixtures("pass_options") +class TestBatching: + __test_path = Path(__file__).parent.resolve() + __data_path = __test_path.joinpath("data") + __harmony_path = __data_path.joinpath("harmony") + + def test_service_invoke(self, temp_output_dir): + in_message_path = self.__harmony_path.joinpath("message.json") + in_message_data = in_message_path.read_text() + + # test with both paged catalogs and un-paged catalogs + for in_catalog_name in ["catalog.json", "catalog0.json"]: + + in_catalog_path = self.__harmony_path.joinpath("source", in_catalog_name) + + test_args = [ + batcher.harmony.cli.__file__, + "--harmony-action", + "invoke", + "--harmony-input", + in_message_data, + "--harmony-source", + str(in_catalog_path), + "--harmony-metadata-dir", + str(temp_output_dir), + "--harmony-data-location", + temp_output_dir.as_uri(), + ] + + test_env = { + "ENV": "dev", + "OAUTH_CLIENT_ID": "", + "OAUTH_UID": "", + "OAUTH_PASSWORD": "", + "OAUTH_REDIRECT_URI": "", + "STAGING_PATH": "", + "STAGING_BUCKET": "", + } + + with patch.object(sys, "argv", test_args), patch.dict(environ, test_env): + batcher.harmony.cli.main() + + # Open the outputs + out_batch_catalog_path = temp_output_dir.joinpath("batch-catalogs.json") + out_batch_catalogs = json.loads(out_batch_catalog_path.read_text()) + + # Go through each batched catalog + batched_files = {0: [], 1: [], 2: []} + for batch_index, catalog in enumerate(out_batch_catalogs): + out_catalog_path = temp_output_dir.joinpath(catalog) + out_catalog = json.loads(out_catalog_path.read_text()) + + for item_meta in out_catalog["links"]: + if item_meta["rel"] == "item": + item_path = temp_output_dir.joinpath(item_meta["href"]).resolve() + + # -- Item Verification -- + item = json.loads(item_path.read_text()) + properties = item["properties"] + assert item["bbox"] + assert properties["start_datetime"] + assert properties["end_datetime"] + + # -- Asset Verification -- + data = item["assets"]["data"] + + # Sanity checks on metadata + assert data["type"] == "application/x-netcdf4" + assert data["roles"] == ["data"] + + batched_files[batch_index].append(Path(urlsplit(data["href"]).path).stem) + + # -- batch file list verification -- + files_dict = { + 0: [ + "TEMPO_NO2_L2_V03_20240601T120101Z_S012G01", + "TEMPO_NO2_L2_V03_20240601T120107Z_S012G02", + ], + 1: [ + "TEMPO_NO2_L2_V03_20240601T120202Z_S013G01", + "TEMPO_NO2_L2_V03_20240601T120209Z_S013G02", + ], + 2: [ + "TEMPO_NO2_L2_V03_20240601T120303Z_S014G01", + "TEMPO_NO2_L2_V03_20240601T120310Z_S014G02", + ], + } + + assert batched_files == files_dict