From 2946140a41f8aef2b4f4f02904d58949d57cb8f8 Mon Sep 17 00:00:00 2001 From: Dan Frank Date: Wed, 13 Mar 2024 11:35:03 -0700 Subject: [PATCH 1/6] Update setup gcloud action version --- .github/workflows/build.yml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 267288b..832f77d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,6 @@ on: - main pull_request: - jobs: test: runs-on: ubuntu-latest @@ -16,19 +15,19 @@ jobs: python-version: ["3.10"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.0 - - uses: google-github-actions/setup-gcloud@v0 - - name: Start GCP Datastore emulator - # NOTE: this emulator is used by the tests below - run: | - gcloud components install --quiet beta cloud-datastore-emulator - gcloud beta emulators datastore start --project foobar --host-port 127.0.0.1:8099 --consistency=1.0 --no-store-on-disk & + - uses: google-github-actions/setup-gcloud@v2 + - name: Start GCP Datastore emulator + # NOTE: this emulator is used by the tests below + run: | + gcloud components install --quiet beta cloud-datastore-emulator + gcloud beta emulators datastore start --project foobar --host-port 127.0.0.1:8099 --consistency=1.0 --no-store-on-disk & - - run: pip install -r requirements.txt -r requirements-dev.txt - - run: pytest --color=yes --datastore_emulated + - run: pip install -r requirements.txt -r requirements-dev.txt + - run: pytest --color=yes --datastore_emulated From 0e4f0ef8a499a4752053fc1d8230a0781489184a Mon Sep 17 00:00:00 2001 From: Dan Frank Date: Wed, 13 Mar 2024 11:36:37 -0700 Subject: [PATCH 2/6] Test in 3.11 as well --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 832f77d..167fb9d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.10", "3.11"] steps: - uses: actions/checkout@v2 From b2a0b60ba4d95a88989c087807382c3a7dc5f8e1 Mon Sep 17 00:00:00 2001 From: Dan Frank Date: Wed, 13 Mar 2024 12:26:40 -0700 Subject: [PATCH 3/6] Change test expectation and name for newer version of fsspec --- articat/tests/utils_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/articat/tests/utils_test.py b/articat/tests/utils_test.py index ba6c79b..c70125b 100644 --- a/articat/tests/utils_test.py +++ b/articat/tests/utils_test.py @@ -44,14 +44,13 @@ def test_download__empty(): download_artifact(a, dst) -def test_download__weird_valid_pattern(): +def test_download__invalid_double_star(): src = get_source_path_that_looks_like_path_from_catalog() src.joinpath("part-1").write_text("sth-2") a = TestFSArtifact(id="sth", files_pattern=f"{src}/**part-1") dst = Path(tempfile.mktemp()) - r = download_artifact(a, dst) - assert dst.joinpath("part-1").read_text() == "sth-2" - assert r == f"{dst}/**part-1" + with pytest.raises(ValueError, match="Invalid pattern"): + download_artifact(a, dst) def test_download__weird_invalid_pattern(): From 89f57eacac59b7efa3dcccd0db47bf07fdfed7a1 Mon Sep 17 00:00:00 2001 From: Dan Frank Date: Wed, 13 Mar 2024 13:50:06 -0700 Subject: [PATCH 4/6] Exclude root dir from glob --- articat/utils/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/articat/utils/utils.py b/articat/utils/utils.py index 8ad9af0..916a909 100644 --- a/articat/utils/utils.py +++ b/articat/utils/utils.py @@ -89,7 +89,9 @@ def download_artifact(artifact: FSArtifact, local_dir: PathType) -> str: prefix = artifact.main_dir # Note: glob results don't have fs scheme prefix_no_scheme = re.sub(FSArtifact._fs_scheme_regex, "", prefix) - to_copy = src_fs.glob(artifact.files_pattern) + # Root directory can be included in glob results if a trailing ** is used (which it often is) + # Don't include it in the list of files to copy, since we create it above when creating local_dir + to_copy = [f for f in src_fs.glob(artifact.files_pattern) if f != prefix] if len(to_copy) == 0: raise ValueError(f"Nothing to copy in `{artifact.files_pattern}`") for f in to_copy: From 984ff7d93da3f6539b2dd5dc2223ccbadc6c0526 Mon Sep 17 00:00:00 2001 From: Dan Frank Date: Wed, 13 Mar 2024 14:15:35 -0700 Subject: [PATCH 5/6] Use StrEnum, test only on py 3.11 --- .github/workflows/build.yml | 2 +- .pre-commit-config.yaml | 9 +++++---- articat/config.py | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 167fb9d..1bb4798 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.11"] steps: - uses: actions/checkout@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f34148..b41e857 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.10 + python: python3.11 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 @@ -23,7 +23,8 @@ repos: rev: v0.991 hooks: - id: mypy - args: ["--strict", "--show-error-codes", "--pretty", "--show-error-context"] + args: + ["--strict", "--show-error-codes", "--pretty", "--show-error-context"] additional_dependencies: - - pandas==1.5.2 - - pydantic==1.10.4 + - pandas==1.5.2 + - pydantic==1.10.4 diff --git a/articat/config.py b/articat/config.py index 4f9c3d5..1e29d87 100644 --- a/articat/config.py +++ b/articat/config.py @@ -4,9 +4,9 @@ import warnings from collections.abc import Mapping, Sequence from configparser import ConfigParser -from enum import Enum +from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, Type +from typing import TYPE_CHECKING, Any from articat.utils.class_or_instance_method import class_or_instance_method @@ -16,7 +16,7 @@ from articat.catalog import Catalog -class ArticatMode(str, Enum): +class ArticatMode(StrEnum): local = "local" gcp_datastore = "gcp_datastore" test = "test" @@ -63,7 +63,7 @@ def register_config( cls, config_paths: Sequence[str] | None = None, config_dict: Mapping[str, Mapping[str, Any]] = {}, - ) -> "Type[ArticatConfig]": + ) -> "type[ArticatConfig]": """ Register configuration from config paths and config dictionary. Config paths are read in order, `config_dict` is applied after config paths. @@ -96,7 +96,7 @@ def mode(self) -> ArticatMode: return ArticatMode(self._config.get("main", "mode", fallback=ArticatMode.local)) @class_or_instance_method - def catalog(self) -> "Type[Catalog]": + def catalog(self) -> "type[Catalog]": """Returns the Catalog implementation for given mode""" if self.mode() == ArticatMode.local: from articat.catalog_local import CatalogLocal From f7b1074d48fa51fd6e0e10915453f72e9b2e7908 Mon Sep 17 00:00:00 2001 From: Dan Frank Date: Thu, 14 Mar 2024 12:33:37 -0700 Subject: [PATCH 6/6] Schedule tests to run every monday too --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1bb4798..0050ab4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,11 @@ on: branches: - main pull_request: + schedule: + # Run every Monday at 18:00 UTC (middle of US working hours) to ensure re-running + # tests with the latest dependencies. + # * is a special character in YAML so you have to quote this string + - cron: "0 18 * * 1" jobs: test: