diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 267288b..0050ab4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,29 +6,33 @@ on: branches: - main pull_request: - + schedule: + # Run every Monday at 18:00 UTC (middle of US working hours) to ensure re-running + # tests with the latest dependencies. + # * is a special character in YAML so you have to quote this string + - cron: "0 18 * * 1" jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.0 - - uses: google-github-actions/setup-gcloud@v0 - - name: Start GCP Datastore emulator - # NOTE: this emulator is used by the tests below - run: | - gcloud components install --quiet beta cloud-datastore-emulator - gcloud beta emulators datastore start --project foobar --host-port 127.0.0.1:8099 --consistency=1.0 --no-store-on-disk & + - uses: google-github-actions/setup-gcloud@v2 + - name: Start GCP Datastore emulator + # NOTE: this emulator is used by the tests below + run: | + gcloud components install --quiet beta cloud-datastore-emulator + gcloud beta emulators datastore start --project foobar --host-port 127.0.0.1:8099 --consistency=1.0 --no-store-on-disk & - - run: pip install -r requirements.txt -r requirements-dev.txt - - run: pytest --color=yes --datastore_emulated + - run: pip install -r requirements.txt -r requirements-dev.txt + - run: pytest --color=yes --datastore_emulated diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f34148..b41e857 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.10 + python: python3.11 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 @@ -23,7 +23,8 @@ repos: rev: v0.991 hooks: - id: mypy - args: ["--strict", "--show-error-codes", "--pretty", "--show-error-context"] + args: + ["--strict", "--show-error-codes", "--pretty", "--show-error-context"] additional_dependencies: - - pandas==1.5.2 - - pydantic==1.10.4 + - pandas==1.5.2 + - pydantic==1.10.4 diff --git a/articat/config.py b/articat/config.py index 4f9c3d5..1e29d87 100644 --- a/articat/config.py +++ b/articat/config.py @@ -4,9 +4,9 @@ import warnings from collections.abc import Mapping, Sequence from configparser import ConfigParser -from enum import Enum +from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, Any, Type +from typing import TYPE_CHECKING, Any from articat.utils.class_or_instance_method import class_or_instance_method @@ -16,7 +16,7 @@ from articat.catalog import Catalog -class ArticatMode(str, Enum): +class ArticatMode(StrEnum): local = "local" gcp_datastore = "gcp_datastore" test = "test" @@ -63,7 +63,7 @@ def register_config( cls, config_paths: Sequence[str] | None = None, config_dict: Mapping[str, Mapping[str, Any]] = {}, - ) -> "Type[ArticatConfig]": + ) -> "type[ArticatConfig]": """ Register configuration from config paths and config dictionary. Config paths are read in order, `config_dict` is applied after config paths. @@ -96,7 +96,7 @@ def mode(self) -> ArticatMode: return ArticatMode(self._config.get("main", "mode", fallback=ArticatMode.local)) @class_or_instance_method - def catalog(self) -> "Type[Catalog]": + def catalog(self) -> "type[Catalog]": """Returns the Catalog implementation for given mode""" if self.mode() == ArticatMode.local: from articat.catalog_local import CatalogLocal diff --git a/articat/tests/utils_test.py b/articat/tests/utils_test.py index ba6c79b..c70125b 100644 --- a/articat/tests/utils_test.py +++ b/articat/tests/utils_test.py @@ -44,14 +44,13 @@ def test_download__empty(): download_artifact(a, dst) -def test_download__weird_valid_pattern(): +def test_download__invalid_double_star(): src = get_source_path_that_looks_like_path_from_catalog() src.joinpath("part-1").write_text("sth-2") a = TestFSArtifact(id="sth", files_pattern=f"{src}/**part-1") dst = Path(tempfile.mktemp()) - r = download_artifact(a, dst) - assert dst.joinpath("part-1").read_text() == "sth-2" - assert r == f"{dst}/**part-1" + with pytest.raises(ValueError, match="Invalid pattern"): + download_artifact(a, dst) def test_download__weird_invalid_pattern(): diff --git a/articat/utils/utils.py b/articat/utils/utils.py index 8ad9af0..916a909 100644 --- a/articat/utils/utils.py +++ b/articat/utils/utils.py @@ -89,7 +89,9 @@ def download_artifact(artifact: FSArtifact, local_dir: PathType) -> str: prefix = artifact.main_dir # Note: glob results don't have fs scheme prefix_no_scheme = re.sub(FSArtifact._fs_scheme_regex, "", prefix) - to_copy = src_fs.glob(artifact.files_pattern) + # Root directory can be included in glob results if a trailing ** is used (which it often is) + # Don't include it in the list of files to copy, since we create it above when creating local_dir + to_copy = [f for f in src_fs.glob(artifact.files_pattern) if f != prefix] if len(to_copy) == 0: raise ValueError(f"Nothing to copy in `{artifact.files_pattern}`") for f in to_copy: