74 use quiltcore not quilt3 (#76)

* add QuiltCore

* replace list_packages

* universal path

* use new quiltcore

* create Volume in Local

* remote_man

* use qc-0.3.1

support package get?

* rearrange methods

* use hashes (partial-hash)

* quiltcore = "^0.3.2"

* auto linter

* lock 0.3.2

* use Manifest for child list

* remove diff / use manifest for child

* disable REPOSITORY_GRYPE

* prefactor commit test

* quiltcore = "^0.3.3"

* poetry update

* 0.9.6

* fix types

* Update poetry.lock

* fix coverage

.github/workflows/mega-linter.yml

@@ -7,7 +7,7 @@ on:  # yamllint disable-line rule:truthy
 permissions: read-all
 env: # Comment env block if you do not want to apply fixes
   APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
-  DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,PYTHON_BANDIT,PYTHON_PYRIGHT,PYTHON_PYLINT,REPOSITORY_SECRETLINT,REPOSITORY_TRIVY,REPOSITORY_TRUFFLEHOG
+  DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,PYTHON_BANDIT,PYTHON_PYRIGHT,PYTHON_PYLINT,REPOSITORY_GRYPE,REPOSITORY_SECRETLINT,REPOSITORY_TRIVY,REPOSITORY_TRUFFLEHOG
   MARKDOWN_MARKDOWNLINT_FILTER_REGEX_EXCLUDE: "tests/example.*ME\\.md" # Exclude example markdown files from markdownlint
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}

.gitignore

@@ -133,3 +133,8 @@ dmypy.json
 
 # Testing artifacts
 data.yaml
+examples
+quilt/
+test/
+test_*
+None*

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 0.9.6 (2023-08-09)
+
+- Start using QuiltCore instead of Quilt3
+- Start with get
+- Incomplete support for put
+
 ## 0.9.5 (2023-06-10)
 
 - Embed package names in attr dict (for parsing)

Makefile

@@ -21,7 +21,7 @@ install:
 update:
 	poetry update
 
-test:
+test: typecheck
 	echo "Testing with WRITE_BUCKET=$(WRITE_BUCKET)"
 	poetry run pytest $(TEST_README) --cov --cov-report xml:coverage.xml
 
@@ -35,6 +35,7 @@ typecheck:
 	poetry run mypy $(PROJECT) tests
 
 coverage:
+	echo "Using WRITE_BUCKET=$(WRITE_BUCKET) | SKIP_LONG_TESTS=$(SKIP_LONG_TESTS)"
 	poetry run pytest --cov --cov-report html:coverage_html
 	open coverage_html/index.html
 
@@ -45,7 +46,7 @@ tag:
 	git tag `poetry version | awk '{print $$2}'`
 	git push --tags
 
-pypi: clean
+pypi: clean clean-git
 	poetry version
 	poetry build
 	poetry publish --dry-run

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "quiltplus"
-version = "0.9.5"
+version = "0.9.6"
 description = "Resource-oriented Python API/CLI for Quilt's decentralized social knowledge platform"
 authors = ["Ernest Prabhakar <[email protected]>"]
 license = "MIT"
@@ -16,11 +16,14 @@ asyncclick = "^8.1.3.4"
 anyio = "^3.7.1"
 isort = "^5.12.0"
 quilt3 = "^5.1.0"
+quiltcore = "^0.3.4"
+# quiltcore = {git = "https://github.com/quiltdata/quiltcore.git", rev = "fix-types"}
 trio = "^0.22.2"
 typing-extensions = "^4.7.1"
 tzlocal = "^5.0.1"
 un-yaml = ">=0.3.1"
 # un-yaml = {git = "https://github.com/data-yaml/un-yaml.git", rev = "main"}
+universal-pathlib = "^0.1.1"
 urllib3 = "<2"
 
 [tool.poetry.group.dev.dependencies]

quiltplus/local.py

@@ -1,15 +1,14 @@
 import logging
-from filecmp import dircmp
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from quilt3.backends import get_package_registry  # type: ignore
+from quiltcore import Volume
 
 from .root import QuiltRoot
 
 
 class QuiltLocal(QuiltRoot):
-
     def __init__(self, attrs: dict):
         """
         Base class to set and manage local sync directory
@@ -23,15 +22,19 @@ def __init__(self, attrs: dict):
         """
         super().__init__(attrs)
         self.local_registry = get_package_registry()
-        logging.debug(f"get_package_registry(): {self.local_registry}")
-        self.make_temp_dir()
+        self.assign_dir()
 
-    def make_temp_dir(self):
-        self.temp_dir = TemporaryDirectory(ignore_cleanup_errors=True)
-        self.last_path = Path(self.temp_dir.name)
+    def assign_dir(self, local_dir: Path | None = None):
+        if not local_dir:
+            self.temp_dir = TemporaryDirectory(ignore_cleanup_errors=True)
+            local_dir = Path(self.temp_dir.name)
+        self.last_path = local_dir
+        self.volume = Volume(self.last_path)
+        return self.last_path
 
     def __del__(self):
-        self.temp_dir.cleanup()
+        if hasattr(self, "temp_dir") and self.temp_dir:
+            self.temp_dir.cleanup()
 
     def check_dir(self, local_dir: Path | None = None):
         """
@@ -80,13 +83,13 @@ def check_dir(self, local_dir: Path | None = None):
         logging.debug(f"check_dir: {dir_var} <= {self.attrs}")
         local_dir = Path(dir_var).resolve()
 
-        self.last_path = local_dir
         if not local_dir.exists():
             logging.warning(f"Path does not exist: {local_dir}")
             local_dir.mkdir(parents=True, exist_ok=True)
         elif not local_dir.is_dir():
             raise ValueError(f"Path is not a directory: {local_dir}")
-        return local_dir
+
+        return self.assign_dir(local_dir)
 
     def check_dir_arg(self, opts: dict):
         local_dir = opts.get(QuiltLocal.K_DIR)
@@ -107,32 +110,6 @@ def local_files(self) -> list[Path]:
     def dest(self):
         return str(self.local_path())
 
-    def local_cache(self) -> Path:
-        base_path = Path(self.local_registry.base.path)
-        logging.debug(f"local_registry.base.path: {base_path}")
-        if not base_path.exists():
-            logging.warning(f"local_cache does not exist: {base_path}")
-        return base_path / self.package
-
-    def _diff(self) -> dict[str, str]:
-        """Compare files in local_dir to local cache"""
-        cache = self.local_cache()
-        if not cache.exists():
-            logging.warning(f"_diff: local_cache[{cache}] does not exist")
-            return {}
-        diff = dircmp(str(cache), self.dest())
-        # logging.debug(f"_diff.diff: {diff}")
-        results = {
-            "add": diff.right_only,
-            "rm": diff.left_only,
-            "touch": diff.diff_files,
-        }
-        return {
-            filename: stage
-            for stage, sublist in results.items()
-            for filename in sublist
-        }
-
     def write_text(self, text: str, file: str, *paths: str):
         dir = self.local_path(*paths)
         p = dir / file

quiltplus/package.py

@@ -2,100 +2,117 @@
 
 import logging
 import shutil
+from pathlib import Path
 
 from quilt3 import Package  # type: ignore
-from typing_extensions import Self, Type
+from quiltcore import Builder, Changes, Manifest
 
 from .local import QuiltLocal
 from .uri import QuiltUri
 
 
 class QuiltPackage(QuiltLocal):
+    ERROR_VOL = "changeset: default volume changed"
     K_STAGE = "stage"
     K_MSG = "message"
-    ERR_MOD = (
-        f"Local files have been modified. Unset --{QuiltLocal.K_FAIL} to overwrite."
-    )
-
-    @classmethod
-    def FromURI(cls: Type[Self], uri: str):
-        attrs = QuiltUri.AttrsFromUri(uri)
-        return cls(attrs)
 
     def __init__(self, attrs: dict):
         super().__init__(attrs)
         self.hash = self.attrs.get(QuiltUri.K_HASH)
 
-    def path_uri(self, sub_path: str):
-        return self.pkg_uri() + f"&{QuiltPackage.K_PTH}=" + sub_path
+    #
+    # Helper Methods
+    #
 
-    async def browse(self):
-        logging.debug(f"browse {self.package} {self.registry} {self.hash}")
-        try:
-            q = (
-                Package.browse(self.package, self.registry, top_hash=self.hash)
-                if self.hash
-                else Package.browse(self.package, self.registry)
-            )
-            return q
-        except Exception as err:
-            logging.error(err)
-        return None
+    def path_uri(self, sub_path: str):
+        return self.pkg_uri() + f"&{self.K_PTH}=" + sub_path
 
-    async def local_pkg(self):
-        q = Package().set_dir(".", path=self.dest())
-        return q
+    def stage_uri(self, stage: str, sub_path: str):
+        return self.path_uri(sub_path).replace(
+            self.PREFIX, f"{self.PREFIX}{self.K_STAGE}+{stage}+"
+        )
 
-    async def remote_pkg(self):
-        return (await self.browse()) or Package()
+    def local_man(self) -> Manifest:
+        try:
+            return self.volume.read_manifest(self.hash)  # type: ignore
+        except Exception:
+            raise ValueError(f"no local manifest for hash: {self.hash}")
+
+    def remote_man(self) -> Manifest:
+        tag = self.attrs.get(QuiltUri.K_TAG, self.namespace.TAG_DEFAULT)
+        opts = {self.domain.KEY_HSH: self.hash} if self.hash else {}
+        print(f"remote_man.pkg: {self.package}:{tag}@{self.hash} -> {opts}")
+        man = self.namespace.get(tag, **opts)
+        if not isinstance(man, Manifest):
+            raise ValueError(f"no remote manifest for hash: {self.hash}")
+        return man
+
+    #
+    # Retrieval Methods
+    #
 
     async def child(self):
-        q = await self.remote_pkg()
-        return list(q.keys())
+        man = self.remote_man()
+        return [entry.name for entry in man.list()]
 
     async def list(self, opts: dict = {}):
         return [self.path_uri(k) for k in await self.child()]
 
-    def stage_uri(self, stage: str, sub_path: str):
-        return self.path_uri(sub_path).replace(
-            QuiltPackage.PREFIX, f"{QuiltPackage.PREFIX}{QuiltPackage.K_STAGE}+{stage}+"
-        )
-
-    async def diff(self, opts: dict = {}):
-        """List files that differ from local_cache()"""
-        self.check_dir_arg(opts)
-        diffs = self._diff()
-        return [self.stage_uri(stage, filename) for filename, stage in diffs.items()]
-
-    def unexpected_loss(self, opts, get=True) -> bool:
-        """Check if _diff and fallible"""
-        modified = [k for k, v in self._diff().items() if v == "touch"]
-        fallible = opts.get(QuiltPackage.K_FAIL, False)
-        return len(modified) > 0 and fallible
-
     async def get(self, opts: dict = {}):
-        """Download package to dest()"""
+        """Download package to dest"""
         dest = self.check_dir_arg(opts)
         logging.debug(f"get dest={dest}: {opts}")
-        if self.unexpected_loss(opts):
-            raise ValueError(f"{dest}: {QuiltPackage.ERR_MOD}\n{self._diff()}")
-        q = await self.remote_pkg()
-        q.fetch(dest=dest)
+        man = self.remote_man()
+        rc = self.volume.put(man)  # TODO: update self.tag
         files = self.local_files()
         return [f"file://{fn}" for fn in files]
-
-    async def commit(self, opts: dict = {}):
-        """Create package in the local registry"""
-        pass
+
+    #
+    # Create/Revise Package
+    #
+
+    def unchanged(self) -> bool: # pragma: no cover
+        if not hasattr(self, "changes"):
+            return True
+        if not isinstance(self.changes, Changes):
+            return True
+
+        return len(self.changeset()) == 0
+
+    def changeset(self) -> Changes: # pragma: no cover
+        vpath = self.volume.path
+        if self.unchanged():
+            self.changes = Changes(vpath)
+        if self.changes.path == vpath:
+            return self.changes
+
+        raise ValueError(f"{self.ERROR_VOL}: {self.changes.path} != {vpath}")
+
+    def commit(self, **kwargs) -> Manifest: # pragma: no cover
+        """
+        Create manifest.
+        Store in the local registry.
+        """
+        src = self.check_dir_arg(kwargs)
+        msg = kwargs.get(self.K_MSG, f"{__name__} {self.Now()} @ {kwargs}")
+        logging.debug(f"commit[{msg}] src={src}")
+        changes = self.changeset()
+        if len(self.changes) == 0:
+            changes.post(src)
+        build = Builder(changes)
+        man = build.post(build.path)
+        if not isinstance(man, Manifest):
+            raise ValueError(f"can not create manifest for {changes}")
+        return man
 
     async def push(self, q: Package, opts: dict):
         """Generic handler for all push methods"""
         dest = self.check_dir_arg(opts)
         kwargs = {
-            QuiltPackage.K_REG: self.registry,
-            QuiltPackage.K_FORCE: not opts.get(QuiltPackage.K_FAIL, False),
-            QuiltPackage.K_MSG: opts.get(
-                QuiltPackage.K_MSG, f"{__name__} {QuiltPackage.Now()} @ {opts}"
+            self.K_REG: self.registry,
+            self.K_FORCE: not opts.get(self.K_FAIL, False),
+            self.K_MSG: opts.get(
+                self.K_MSG, f"{__name__} {self.Now()} @ {opts}"
             ),
         }
         logging.debug(f"push dest={dest}: {opts}\n{kwargs}")
@@ -117,6 +134,30 @@ async def patch(self, opts: dict = {}):
         q = await self.remote_pkg()  # reset to latest
         return await self.push(q, opts)
 
+    #
+    # Legacy Methods
+    #
+
+    async def browse(self):
+        logging.debug(f"browse {self.package} {self.registry} {self.hash}")
+        try:
+            q = (
+                Package.browse(self.package, self.registry, top_hash=self.hash)
+                if self.hash
+                else Package.browse(self.package, self.registry)
+            )
+            return q
+        except Exception as err:
+            logging.error(err)
+        return None
+
+    async def local_pkg(self):
+        q = Package().set_dir(".", path=self.dest())
+        return q
+
+    async def remote_pkg(self):
+        return (await self.browse()) or Package()
+
     def delete(self):  # remove local cache
         return shutil.rmtree(self.last_path)