Update CI and switch to python 3.11

.github/workflows/build.yml

@@ -6,29 +6,33 @@ on:
     branches:
       - main
   pull_request:
-
+  schedule:
+    # Run every Monday at 18:00 UTC (middle of US working hours) to ensure re-running
+    # tests with the latest dependencies.
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 18 * * 1"
 
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
 
-    - uses: pre-commit/[email protected]
+      - uses: pre-commit/[email protected]
 
-    - uses: google-github-actions/setup-gcloud@v0
-    - name: Start GCP Datastore emulator
-      # NOTE: this emulator is used by the tests below
-      run: |
-        gcloud components install --quiet beta cloud-datastore-emulator
-        gcloud beta emulators datastore start --project foobar --host-port 127.0.0.1:8099 --consistency=1.0 --no-store-on-disk &
+      - uses: google-github-actions/setup-gcloud@v2
+      - name: Start GCP Datastore emulator
+        # NOTE: this emulator is used by the tests below
+        run: |
+          gcloud components install --quiet beta cloud-datastore-emulator
+          gcloud beta emulators datastore start --project foobar --host-port 127.0.0.1:8099 --consistency=1.0 --no-store-on-disk &
 
-    - run: pip install -r requirements.txt -r requirements-dev.txt
-    - run: pytest --color=yes --datastore_emulated
+      - run: pip install -r requirements.txt -r requirements-dev.txt
+      - run: pytest --color=yes --datastore_emulated

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-  python: python3.10
+  python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
@@ -23,7 +23,8 @@ repos:
     rev: v0.991
     hooks:
       - id: mypy
-        args: ["--strict", "--show-error-codes", "--pretty", "--show-error-context"]
+        args:
+          ["--strict", "--show-error-codes", "--pretty", "--show-error-context"]
         additional_dependencies:
-        - pandas==1.5.2
-        - pydantic==1.10.4
+          - pandas==1.5.2
+          - pydantic==1.10.4

articat/config.py

@@ -4,9 +4,9 @@
 import warnings
 from collections.abc import Mapping, Sequence
 from configparser import ConfigParser
-from enum import Enum
+from enum import StrEnum
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Type
+from typing import TYPE_CHECKING, Any
 
 from articat.utils.class_or_instance_method import class_or_instance_method
 
@@ -16,7 +16,7 @@
     from articat.catalog import Catalog
 
 
-class ArticatMode(str, Enum):
+class ArticatMode(StrEnum):
     local = "local"
     gcp_datastore = "gcp_datastore"
     test = "test"
@@ -63,7 +63,7 @@ def register_config(
         cls,
         config_paths: Sequence[str] | None = None,
         config_dict: Mapping[str, Mapping[str, Any]] = {},
-    ) -> "Type[ArticatConfig]":
+    ) -> "type[ArticatConfig]":
         """
         Register configuration from config paths and config dictionary. Config
         paths are read in order, `config_dict` is applied after config paths.
@@ -96,7 +96,7 @@ def mode(self) -> ArticatMode:
         return ArticatMode(self._config.get("main", "mode", fallback=ArticatMode.local))
 
     @class_or_instance_method
-    def catalog(self) -> "Type[Catalog]":
+    def catalog(self) -> "type[Catalog]":
         """Returns the Catalog implementation for given mode"""
         if self.mode() == ArticatMode.local:
             from articat.catalog_local import CatalogLocal

articat/tests/utils_test.py

@@ -44,14 +44,13 @@ def test_download__empty():
         download_artifact(a, dst)
 
 
-def test_download__weird_valid_pattern():
+def test_download__invalid_double_star():
     src = get_source_path_that_looks_like_path_from_catalog()
     src.joinpath("part-1").write_text("sth-2")
     a = TestFSArtifact(id="sth", files_pattern=f"{src}/**part-1")
     dst = Path(tempfile.mktemp())
-    r = download_artifact(a, dst)
-    assert dst.joinpath("part-1").read_text() == "sth-2"
-    assert r == f"{dst}/**part-1"
+    with pytest.raises(ValueError, match="Invalid pattern"):
+        download_artifact(a, dst)
 
 
 def test_download__weird_invalid_pattern():

articat/utils/utils.py

@@ -89,7 +89,9 @@ def download_artifact(artifact: FSArtifact, local_dir: PathType) -> str:
     prefix = artifact.main_dir
     # Note: glob results don't have fs scheme
     prefix_no_scheme = re.sub(FSArtifact._fs_scheme_regex, "", prefix)
-    to_copy = src_fs.glob(artifact.files_pattern)
+    # Root directory can be included in glob results if a trailing ** is used (which it often is)
+    # Don't include it in the list of files to copy, since we create it above when creating local_dir
+    to_copy = [f for f in src_fs.glob(artifact.files_pattern) if f != prefix]
     if len(to_copy) == 0:
         raise ValueError(f"Nothing to copy in `{artifact.files_pattern}`")
     for f in to_copy: