From 49d36a226c89106cd3cb8536119c3ae0b46d1fe5 Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Wed, 8 Sep 2021 18:08:59 -0400
Subject: [PATCH 1/7] WIP: Virtual packages

---
 conda_lock/conda_lock.py              |  35 ++++-
 conda_lock/src_parser/__init__.py     |  30 ++--
 conda_lock/virtual_package.py         | 195 ++++++++++++++++++++++++++
 mypy.ini                              |   8 ++
 requirements.txt                      |   1 +
 tests/test-cuda/environment.yaml      |   6 +
 tests/test-cuda/virtual-packages.yaml |   4 +
 tests/test_conda_lock.py              |   1 -
 8 files changed, 262 insertions(+), 18 deletions(-)
 create mode 100644 conda_lock/virtual_package.py
 create mode 100644 mypy.ini
 create mode 100644 tests/test-cuda/environment.yaml
 create mode 100644 tests/test-cuda/virtual-packages.yaml

diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py
index 9774f0bbb..8b7abf876 100644
--- a/conda_lock/conda_lock.py
+++ b/conda_lock/conda_lock.py
@@ -43,6 +43,11 @@
 from conda_lock.src_parser.environment_yaml import parse_environment_file
 from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
 from conda_lock.src_parser.pyproject_toml import parse_pyproject_toml
+from conda_lock.virtual_package import (
+    FakeRepoData,
+    default_virtual_package_repodata,
+    virtual_package_repo_from_specification,
+)
 
 
 DEFAULT_FILES = [pathlib.Path("environment.yml")]
@@ -152,7 +157,10 @@ def conda_env_override(platform) -> Dict[str, str]:
 
 
 def solve_specs_for_arch(
-    conda: PathLike, channels: Sequence[str], specs: List[str], platform: str
+    conda: PathLike,
+    channels: Sequence[str],
+    specs: List[str],
+    platform: str,
 ) -> dict:
     args: MutableSequence[PathLike] = [
         str(conda),
@@ -167,6 +175,7 @@ def solve_specs_for_arch(
         args.extend(shlex.split(conda_flags))
     if channels:
         args.append("--override-channels")
+
     for channel in channels:
         args.extend(["--channel", channel])
         if channel == "defaults" and platform in {"win-64", "win-32"}:
@@ -359,6 +368,7 @@ def make_lock_specs(
     include_dev_dependencies: bool = True,
     channel_overrides: Optional[Sequence[str]] = None,
     extras: Optional[AbstractSet[str]] = None,
+    virtual_package_repo: FakeRepoData,
 ) -> Dict[str, LockSpecification]:
     """Generate the lockfile specs from a set of input src_files"""
     res = {}
@@ -375,6 +385,7 @@ def make_lock_specs(
             channels = list(channel_overrides)
         else:
             channels = lock_spec.channels
+        lock_spec.virtual_package_repo = virtual_package_repo
         lock_spec.channels = channels
         res[plat] = lock_spec
     return res
@@ -390,6 +401,7 @@ def make_lock_files(
     filename_template: Optional[str] = None,
     check_spec_hash: bool = False,
     extras: Optional[AbstractSet[str]] = None,
+    virtual_package_spec: Optional[pathlib.Path] = None,
 ):
     """Generate the lock files for the given platforms from the src file provided
 
@@ -430,12 +442,21 @@ def make_lock_files(
                 )
                 sys.exit(1)
 
+    # initialize virtual package fake
+    if virtual_package_spec and virtual_package_spec.exists():
+        virtual_package_repo = virtual_package_repo_from_specification(
+            virtual_package_spec
+        )
+    else:
+        virtual_package_repo = default_virtual_package_repodata()
+
     lock_specs = make_lock_specs(
         platforms=platforms,
         src_files=src_files,
         include_dev_dependencies=include_dev_dependencies,
         channel_overrides=channel_overrides,
         extras=extras,
+        virtual_package_repo=virtual_package_repo,
     )
 
     for plat, lock_spec in lock_specs.items():
@@ -467,7 +488,6 @@ def make_lock_files(
 
             print(f"Generating lockfile(s) for {plat}...", file=sys.stderr)
             lockfile_contents = create_lockfile_from_spec(
-                channels=lock_spec.channels,
                 conda=conda,
                 spec=lock_spec,
                 kind=kind,
@@ -502,15 +522,16 @@ def is_micromamba(conda: PathLike) -> bool:
 
 def create_lockfile_from_spec(
     *,
-    channels: Sequence[str],
     conda: PathLike,
     spec: LockSpecification,
     kind: str,
 ) -> List[str]:
+    assert spec.virtual_package_repo is not None
+    virtual_package_channel = spec.virtual_package_repo.channel_url
     dry_run_install = solve_specs_for_arch(
         conda=conda,
         platform=spec.platform,
-        channels=channels,
+        channels=[*spec.channels, virtual_package_channel],
         specs=spec.specs,
     )
     logging.debug("dry_run_install:\n%s", dry_run_install)
@@ -526,11 +547,12 @@ def create_lockfile_from_spec(
         lockfile_contents.extend(
             [
                 "channels:",
-                *(f"  - {channel}" for channel in channels),
+                *(f"  - {channel}" for channel in spec.channels),
                 "dependencies:",
                 *(
                     f'  - {pkg["name"]}={pkg["version"]}={pkg["build_string"]}'
                     for pkg in link_actions
+                    # TODO: exclude injected virtual packages
                 ),
             ]
         )
@@ -546,7 +568,6 @@ def create_lockfile_from_spec(
                 link[
                     "url_base"
                 ] = f"{link['base_url']}/{link['platform']}/{link['dist_name']}"
-
             link["url"] = f"{link['url_base']}.tar.bz2"
             link["url_conda"] = f"{link['url_base']}.conda"
         link_dists = {link["dist_name"] for link in link_actions}
@@ -563,7 +584,7 @@ def create_lockfile_from_spec(
                     x for x in link_actions if x["dist_name"] in non_fetch_packages
                 ],
                 platform=spec.platform,
-                channels=channels,
+                channels=spec.channels,
             ):
                 dist_name = fn_to_dist_name(search_res["fn"])
                 fetch_by_dist_name[dist_name] = search_res
diff --git a/conda_lock/src_parser/__init__.py b/conda_lock/src_parser/__init__.py
index c6cd15393..f4741c59c 100644
--- a/conda_lock/src_parser/__init__.py
+++ b/conda_lock/src_parser/__init__.py
@@ -1,22 +1,32 @@
 import hashlib
 import json
 
-from typing import List
+from typing import List, Optional
+
+from conda_lock.virtual_package import FakeRepoData
 
 
 class LockSpecification:
-    def __init__(self, specs: List[str], channels: List[str], platform: str):
+    def __init__(
+        self,
+        specs: List[str],
+        channels: List[str],
+        platform: str,
+        virtual_package_repo: Optional[FakeRepoData] = None,
+    ):
         self.specs = specs
         self.channels = channels
         self.platform = platform
+        self.virtual_package_repo = virtual_package_repo
 
     def input_hash(self) -> str:
-        env_spec = json.dumps(
-            {
-                "channels": self.channels,
-                "platform": self.platform,
-                "specs": sorted(self.specs),
-            },
-            sort_keys=True,
-        )
+        data: dict = {
+            "channels": self.channels,
+            "platform": self.platform,
+            "specs": sorted(self.specs),
+        }
+        if self.virtual_package_repo is not None:
+            data["virtual_package_hash"] = self.virtual_package_repo.all_repodata
+
+        env_spec = json.dumps(data, sort_keys=True)
         return hashlib.sha256(env_spec.encode("utf-8")).hexdigest()
diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py
new file mode 100644
index 000000000..8386fcce5
--- /dev/null
+++ b/conda_lock/virtual_package.py
@@ -0,0 +1,195 @@
+import atexit
+import json
+import logging
+import os
+import pathlib
+
+from collections import defaultdict
+from typing import Dict, Iterable, List, Optional, Set, Tuple
+
+from pydantic import BaseModel, Field, validator
+
+
+logger = logging.getLogger(__name__)
+
+# datetime.datetime(2020, 1, 1).timestamp()
+DEFAULT_TIME = 1577854800000
+
+
+class FakePackage(BaseModel):
+    class Config:
+        """pydantic config."""
+
+        allow_mutation = False
+        frozen = True
+
+    name: str
+    version: str = "1.0"
+    build_string: str = ""
+    build_number: int = 0
+    noarch: str = ""
+    depends: Tuple[str, ...] = Field(default_factory=tuple)
+    timestamp: int = DEFAULT_TIME
+
+    def to_repodata_entry(self):
+        out = self.dict()
+        if self.build_string:
+            build = f"{self.build_string}_{self.build_number}"
+        else:
+            build = f"{self.build_number}"
+        out["depends"] = list(out["depends"])
+        out["build"] = build
+        fname = f"{self.name}-{self.version}-{build}.tar.bz2"
+        return fname, out
+
+
+class FakeRepoData:
+    def __init__(self, base_dir: pathlib.Path):
+        self.base_path = base_dir
+        self.packages_by_subdir: Dict[FakePackage, Set[str]] = defaultdict(set)
+        self.all_subdirs = {
+            "noarch",
+            "linux-aarch64",
+            "linux-ppc64le",
+            "linux-64",
+            "osx-64",
+            "osx-arm64",
+            "win-64",
+        }
+        self.all_repodata: Dict[str, dict] = {}
+        self.hash: Optional[str] = None
+
+    @property
+    def channel_url(self):
+        return f"file://{str(self.base_path.absolute())}"
+
+    def add_package(self, package: FakePackage, subdirs: Iterable[str] = ()):
+        subdirs = frozenset(subdirs)
+        if not subdirs:
+            subdirs = frozenset(["noarch"])
+        self.packages_by_subdir[package].update(subdirs)
+
+    def _write_subdir(self, subdir: str) -> dict:
+        packages: dict = {}
+        out = {"info": {"subdir": subdir}, "packages": packages}
+        for pkg, subdirs in self.packages_by_subdir.items():
+            if subdir not in subdirs:
+                continue
+            fname, info_dict = pkg.to_repodata_entry()
+            info_dict["subdir"] = subdir
+            packages[fname] = info_dict
+
+        (self.base_path / subdir).mkdir(exist_ok=True)
+        content = json.dumps(out, sort_keys=True)
+        (self.base_path / subdir / "repodata.json").write_text(content)
+        return out
+
+    def write(self) -> None:
+        for subdirs in self.packages_by_subdir.values():
+            self.all_subdirs.update(subdirs)
+
+        for subdir in sorted(self.all_subdirs):
+            repodata = self._write_subdir(subdir)
+            self.all_repodata[subdir] = repodata
+
+        logger.info("Wrote fake repodata to %s", self.base_path)
+        import glob
+
+        for filename in glob.iglob(str(self.base_path / "**"), recursive=True):
+            logger.info(filename)
+        logger.info("repo: %s", self.channel_url)
+
+
+def _init_fake_repodata() -> FakeRepoData:
+    import shutil
+    import tempfile
+
+    # tmp directory in github actions
+    runner_tmp = os.environ.get("RUNNER_TEMP")
+    tmp_dir = tempfile.mkdtemp(dir=runner_tmp)
+
+    if not runner_tmp:
+        # no need to bother cleaning up on CI
+        def clean():
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+
+        atexit.register(clean)
+
+    tmp_path = pathlib.Path(tmp_dir)
+    repodata = FakeRepoData(tmp_path)
+    return repodata
+
+
+OSX_VERSIONS_X86 = ["10.15"]
+OSX_VERSIONS_X68_ARM64 = ["11.0"]
+OSX_VERSIONS_ARM64: List[str] = []
+
+
+def default_virtual_package_repodata() -> FakeRepoData:
+    repodata = _init_fake_repodata()
+    fake_packages = [
+        FakePackage(name="__glibc", version="2.17"),
+        FakePackage(name="__cuda", version="11.4"),
+    ]
+    for pkg in fake_packages:
+        repodata.add_package(pkg)
+
+    for osx_ver in OSX_VERSIONS_X86:
+        package = FakePackage(name="__osx", version=osx_ver)
+        repodata.add_package(package, subdirs=["osx-64"])
+    for osx_ver in OSX_VERSIONS_X68_ARM64:
+        package = FakePackage(name="__osx", version=osx_ver)
+        repodata.add_package(package, subdirs=["osx-64", "osx-arm64"])
+    for osx_ver in OSX_VERSIONS_ARM64:
+        package = FakePackage(name="__osx", version=osx_ver)
+        repodata.add_package(package, subdirs=["osx-arm64"])
+    repodata.write()
+    return repodata
+
+
+class VirtualPackageSpecSubdir(BaseModel):
+    packages: Dict[str, str]
+
+    @validator("packages")
+    def validate_packages(cls, v: Dict[str, str]):
+        for package_name in v:
+            if not package_name.startswith("__"):
+                raise ValueError(f"{package_name} is not a virtual package!")
+        return v
+
+
+class VirtualPackageSpec(BaseModel):
+    subdirs: Dict[str, VirtualPackageSpecSubdir]
+
+
+def virtual_package_repo_from_specification(
+    virtual_package_spec_file: pathlib.Path,
+) -> FakeRepoData:
+    import yaml
+
+    with virtual_package_spec_file.open("r") as fp:
+        data = yaml.safe_load(fp)
+
+    spec = VirtualPackageSpec.parse_obj(data)
+
+    repodata = _init_fake_repodata()
+    for subdir, subdir_spec in spec.subdirs.items():
+        for virtual_package, version in subdir_spec.packages.items():
+            repodata.add_package(
+                FakePackage(name=virtual_package, version=version), subdirs=[subdir]
+            )
+    repodata.write()
+    return repodata
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    fil = (
+        pathlib.Path(__file__).parent.parent
+        / "tests"
+        / "test-cuda"
+        / "virtual-packages.yaml"
+    )
+    rd = virtual_package_repo_from_specification(fil)
+    print(rd)
+    print((rd.base_path / "linux-64" / "repodata.json").read_text())
diff --git a/mypy.ini b/mypy.ini
new file mode 100644
index 000000000..5cb02c2ee
--- /dev/null
+++ b/mypy.ini
@@ -0,0 +1,8 @@
+[mypy]
+plugins = pydantic.mypy
+
+[pydantic-mypy]
+init_forbid_extra = True
+init_typed = True
+warn_required_dynamic_aliases = True
+warn_untyped_fields = True
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index e3e632173..ab405b168 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ toml
 ensureconda>=1.1.0
 click
 click-default-group
+pydantic
\ No newline at end of file
diff --git a/tests/test-cuda/environment.yaml b/tests/test-cuda/environment.yaml
new file mode 100644
index 000000000..e56b319dc
--- /dev/null
+++ b/tests/test-cuda/environment.yaml
@@ -0,0 +1,6 @@
+name: cuda
+channels:
+  - conda-forge
+dependencies:
+  - cudatoolkit =11.0
+  - cudnn =8.0
\ No newline at end of file
diff --git a/tests/test-cuda/virtual-packages.yaml b/tests/test-cuda/virtual-packages.yaml
new file mode 100644
index 000000000..fec6f5bc4
--- /dev/null
+++ b/tests/test-cuda/virtual-packages.yaml
@@ -0,0 +1,4 @@
+subdirs:
+  linux-64:
+    packages:
+      __glibc: 2.14
\ No newline at end of file
diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py
index 1ea4c4330..3303ef973 100644
--- a/tests/test_conda_lock.py
+++ b/tests/test_conda_lock.py
@@ -208,7 +208,6 @@ def test_poetry_version_parsing_constraints(package, version, url_pattern):
     )
     lockfile_contents = create_lockfile_from_spec(
         conda=_conda_exe,
-        channels=spec.channels,
         spec=spec,
         kind="explicit",
     )

From fba1d6634242246fcc56e80449ce8fe213ba07e3 Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Wed, 8 Sep 2021 22:02:49 -0400
Subject: [PATCH 2/7] WIP: Continued

---
 conda_lock/conda_lock.py              | 44 ++++++++++++++++++++-------
 conda_lock/virtual_package.py         |  6 ++--
 setup.cfg                             |  1 +
 tests/test-cuda/conda-linux-64.lock   | 11 +++++++
 tests/test-cuda/environment.yml       |  6 ++++
 tests/test-cuda/virtual-packages.yaml |  2 +-
 6 files changed, 55 insertions(+), 15 deletions(-)
 create mode 100644 tests/test-cuda/conda-linux-64.lock
 create mode 100644 tests/test-cuda/environment.yml

diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py
index 8b7abf876..529966c33 100644
--- a/conda_lock/conda_lock.py
+++ b/conda_lock/conda_lock.py
@@ -50,8 +50,8 @@
 )
 
 
+logger = logging.getLogger(__name__)
 DEFAULT_FILES = [pathlib.Path("environment.yml")]
-
 PathLike = Union[str, pathlib.Path]
 
 # Captures basic auth credentials, if they exists, in the second capture group.
@@ -594,6 +594,8 @@ def create_lockfile_from_spec(
                 fn_to_dist_name(pkg["fn"]) if is_micromamba(conda) else pkg["dist_name"]
             )
             url = fetch_by_dist_name[dist_name]["url"]
+            if url.startswith(virtual_package_channel):
+                continue
             md5 = fetch_by_dist_name[dist_name]["md5"]
             lockfile_contents.append(f"{url}#{md5}")
 
@@ -785,6 +787,7 @@ def run_lock(
     kinds: Optional[List[str]] = None,
     check_input_hash: bool = False,
     extras: Optional[AbstractSet[str]] = None,
+    virtual_package_spec: Optional[pathlib.Path] = None,
 ) -> None:
     if environment_files == DEFAULT_FILES:
         long_ext_file = pathlib.Path("environment.yaml")
@@ -804,6 +807,7 @@ def run_lock(
         kinds=kinds or DEFAULT_KINDS,
         check_spec_hash=check_input_hash,
         extras=extras,
+        virtual_package_spec=virtual_package_spec,
     )
 
 
@@ -891,16 +895,12 @@ def main():
     default="INFO",
     type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
 )
-# @click.option(
-#     "-m",
-#     "--mode",
-#     type=click.Choice(["default", "docker"], case_sensitive=True),
-#     default="default",
-#     help="""
-#             Run this conda-lock in an isolated docker container.  This may be
-#             required to account for some issues where conda-lock conflicts with
-#             existing condarc configurations.""",
-# )
+@click.option('--pdb', is_flag=True, help="Drop into a postmortem debugger if conda-lock crashes")
+@click.option(
+    '--virtual-package-spec', 
+    type=click.Path(),
+    help='Specify a set of virtual packages to use.',
+)
 def lock(
     conda,
     mamba,
@@ -915,6 +915,8 @@ def lock(
     extras,
     check_input_hash: bool,
     log_level,
+    pdb,
+    virtual_package_spec,
 ):
     """Generate fully reproducible lock files for conda environments.
 
@@ -929,6 +931,25 @@ def lock(
         timestamp: The approximate timestamp of the output file in ISO8601 basic format.
     """
     logging.basicConfig(level=log_level)
+
+    if pdb:
+        def handle_exception(exc_type, exc_value, exc_traceback):
+            import pdb
+            pdb.post_mortem(exc_traceback)      
+    
+        sys.excepthook = handle_exception
+
+    if not virtual_package_spec:
+        candidates = [
+            pathlib.Path('virtual-packages.yml'),
+            pathlib.Path('virtual-packages.yaml'),
+        ]
+        for c in candidates:
+            if c.exists():
+                logger.info("Using virtual packages from %s", c)
+                virtual_package_spec = c
+                break
+
     files = [pathlib.Path(file) for file in files]
     extras = set(extras)
     lock_func = partial(
@@ -942,6 +963,7 @@ def lock(
         channel_overrides=channel_overrides,
         kinds=kind,
         extras=extras,
+        virtual_package_spec=virtual_package_spec,
     )
     if strip_auth:
         with tempfile.TemporaryDirectory() as tempdir:
diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py
index 8386fcce5..5568ac19e 100644
--- a/conda_lock/virtual_package.py
+++ b/conda_lock/virtual_package.py
@@ -92,12 +92,12 @@ def write(self) -> None:
             repodata = self._write_subdir(subdir)
             self.all_repodata[subdir] = repodata
 
-        logger.info("Wrote fake repodata to %s", self.base_path)
+        logger.debug("Wrote fake repodata to %s", self.base_path)
         import glob
 
         for filename in glob.iglob(str(self.base_path / "**"), recursive=True):
-            logger.info(filename)
-        logger.info("repo: %s", self.channel_url)
+            logger.debug(filename)
+        logger.debug("repo: %s", self.channel_url)
 
 
 def _init_fake_repodata() -> FakeRepoData:
diff --git a/setup.cfg b/setup.cfg
index 427549f0e..37303c74c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,6 +36,7 @@ install_requires =
     ensureconda >=1.1
     click
     click-default-group
+    pydantic >=1.8.1
 python_requires = >=3.6
 packages = find:
 setup_requires =
diff --git a/tests/test-cuda/conda-linux-64.lock b/tests/test-cuda/conda-linux-64.lock
new file mode 100644
index 000000000..3e4241654
--- /dev/null
+++ b/tests/test-cuda/conda-linux-64.lock
@@ -0,0 +1,11 @@
+# Generated by conda-lock.
+# platform: linux-64
+# input_hash: 1fbf01fc1830a11428199dab0c1c0f63db3468919e3434f4672cced029aea2cb
+@EXPLICIT
+https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.1.0-h56837e0_8.tar.bz2#930957b6bff66cfd539ada080c5ca3e8
+https://conda.anaconda.org/conda-forge/linux-64/libgomp-11.1.0-hc902ee8_8.tar.bz2#f2dd961d1ae80d9d81b3d5068807f11b
+https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2#561e277319a41d4f24f5c05a9ef63c04
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-11.1.0-hc902ee8_8.tar.bz2#da6221956ce8582d8e71acc16dfe4c3e
+https://conda.anaconda.org/conda-forge/linux-64/cudatoolkit-11.0.3-h15472ef_8.tar.bz2#afed8db569f6974fdcd5e9f362985bf2
+https://conda.anaconda.org/conda-forge/linux-64/cudnn-8.0.5.39-ha5ca753_1.tar.bz2#8a89f290d4175f8f11c5ddba81fd16d6
diff --git a/tests/test-cuda/environment.yml b/tests/test-cuda/environment.yml
new file mode 100644
index 000000000..e56b319dc
--- /dev/null
+++ b/tests/test-cuda/environment.yml
@@ -0,0 +1,6 @@
+name: cuda
+channels:
+  - conda-forge
+dependencies:
+  - cudatoolkit =11.0
+  - cudnn =8.0
\ No newline at end of file
diff --git a/tests/test-cuda/virtual-packages.yaml b/tests/test-cuda/virtual-packages.yaml
index fec6f5bc4..37d58b3f0 100644
--- a/tests/test-cuda/virtual-packages.yaml
+++ b/tests/test-cuda/virtual-packages.yaml
@@ -1,4 +1,4 @@
 subdirs:
   linux-64:
     packages:
-      __glibc: 2.14
\ No newline at end of file
+      __glibc: 2.17
\ No newline at end of file

From cfc92cf93f0d43fd4d195b1eb3faca6b58101b9f Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Thu, 9 Sep 2021 10:19:25 -0400
Subject: [PATCH 3/7] Add a test suite for checking virtual packages

---
 conda_lock/conda_lock.py                      | 120 ++++++++++--------
 conda_lock/virtual_package.py                 |  22 ++++
 tests/test-cuda/conda-linux-64.lock           |   2 +-
 tests/test-cuda/conda-linux-64.lock.yml       |  15 +++
 tests/test-cuda/environment.yml               |   6 -
 .../test-cuda/virtual-packages-old-glibc.yaml |   4 +
 tests/test_conda_lock.py                      |  83 +++++++++---
 7 files changed, 170 insertions(+), 82 deletions(-)
 create mode 100644 tests/test-cuda/conda-linux-64.lock.yml
 delete mode 100644 tests/test-cuda/environment.yml
 create mode 100644 tests/test-cuda/virtual-packages-old-glibc.yaml

diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py
index 529966c33..bc333ab0e 100644
--- a/conda_lock/conda_lock.py
+++ b/conda_lock/conda_lock.py
@@ -450,58 +450,61 @@ def make_lock_files(
     else:
         virtual_package_repo = default_virtual_package_repodata()
 
-    lock_specs = make_lock_specs(
-        platforms=platforms,
-        src_files=src_files,
-        include_dev_dependencies=include_dev_dependencies,
-        channel_overrides=channel_overrides,
-        extras=extras,
-        virtual_package_repo=virtual_package_repo,
-    )
+    with virtual_package_repo:
+        lock_specs = make_lock_specs(
+            platforms=platforms,
+            src_files=src_files,
+            include_dev_dependencies=include_dev_dependencies,
+            channel_overrides=channel_overrides,
+            extras=extras,
+            virtual_package_repo=virtual_package_repo,
+        )
 
-    for plat, lock_spec in lock_specs.items():
-        for kind in kinds:
-            if filename_template:
-                context = {
-                    "platform": lock_spec.platform,
-                    "dev-dependencies": str(include_dev_dependencies).lower(),
-                    # legacy key
-                    "spec-hash": lock_spec.input_hash(),
-                    "input-hash": lock_spec.input_hash(),
-                    "version": pkg_resources.get_distribution("conda_lock").version,
-                    "timestamp": datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"),
-                }
-
-                filename = filename_template.format(**context)
-            else:
-                filename = f"conda-{lock_spec.platform}.lock"
-
-            lockfile = pathlib.Path(filename)
-            if lockfile.exists() and check_spec_hash:
-                existing_spec_hash = extract_input_hash(lockfile.read_text())
-                if existing_spec_hash == lock_spec.input_hash():
-                    print(
-                        f"Spec hash already locked for {plat}. Skipping",
-                        file=sys.stderr,
-                    )
-                    continue
-
-            print(f"Generating lockfile(s) for {plat}...", file=sys.stderr)
-            lockfile_contents = create_lockfile_from_spec(
-                conda=conda,
-                spec=lock_spec,
-                kind=kind,
-            )
+        for plat, lock_spec in lock_specs.items():
+            for kind in kinds:
+                if filename_template:
+                    context = {
+                        "platform": lock_spec.platform,
+                        "dev-dependencies": str(include_dev_dependencies).lower(),
+                        # legacy key
+                        "spec-hash": lock_spec.input_hash(),
+                        "input-hash": lock_spec.input_hash(),
+                        "version": pkg_resources.get_distribution("conda_lock").version,
+                        "timestamp": datetime.datetime.utcnow().strftime(
+                            "%Y%m%dT%H%M%SZ"
+                        ),
+                    }
+
+                    filename = filename_template.format(**context)
+                else:
+                    filename = f"conda-{lock_spec.platform}.lock"
+
+                lockfile = pathlib.Path(filename)
+                if lockfile.exists() and check_spec_hash:
+                    existing_spec_hash = extract_input_hash(lockfile.read_text())
+                    if existing_spec_hash == lock_spec.input_hash():
+                        print(
+                            f"Spec hash already locked for {plat}. Skipping",
+                            file=sys.stderr,
+                        )
+                        continue
+
+                print(f"Generating lockfile(s) for {plat}...", file=sys.stderr)
+                lockfile_contents = create_lockfile_from_spec(
+                    conda=conda,
+                    spec=lock_spec,
+                    kind=kind,
+                )
 
-            filename += KIND_FILE_EXT[kind]
-            with open(filename, "w") as fo:
-                fo.write("\n".join(lockfile_contents) + "\n")
+                filename += KIND_FILE_EXT[kind]
+                with open(filename, "w") as fo:
+                    fo.write("\n".join(lockfile_contents) + "\n")
 
-            print(
-                f" - Install lock using {'(see warning below)' if kind == 'env' else ''}:",
-                KIND_USE_TEXT[kind].format(lockfile=filename),
-                file=sys.stderr,
-            )
+                print(
+                    f" - Install lock using {'(see warning below)' if kind == 'env' else ''}:",
+                    KIND_USE_TEXT[kind].format(lockfile=filename),
+                    file=sys.stderr,
+                )
 
     if "env" in kinds:
         print(
@@ -877,6 +880,7 @@ def main():
     help="Strip the basic auth credentials from the lockfile.",
 )
 @click.option(
+    "-e",
     "--extras",
     default=[],
     type=str,
@@ -895,11 +899,13 @@ def main():
     default="INFO",
     type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
 )
-@click.option('--pdb', is_flag=True, help="Drop into a postmortem debugger if conda-lock crashes")
 @click.option(
-    '--virtual-package-spec', 
+    "--pdb", is_flag=True, help="Drop into a postmortem debugger if conda-lock crashes"
+)
+@click.option(
+    "--virtual-package-spec",
     type=click.Path(),
-    help='Specify a set of virtual packages to use.',
+    help="Specify a set of virtual packages to use.",
 )
 def lock(
     conda,
@@ -933,16 +939,18 @@ def lock(
     logging.basicConfig(level=log_level)
 
     if pdb:
+
         def handle_exception(exc_type, exc_value, exc_traceback):
             import pdb
-            pdb.post_mortem(exc_traceback)      
-    
+
+            pdb.post_mortem(exc_traceback)
+
         sys.excepthook = handle_exception
 
     if not virtual_package_spec:
         candidates = [
-            pathlib.Path('virtual-packages.yml'),
-            pathlib.Path('virtual-packages.yaml'),
+            pathlib.Path("virtual-packages.yml"),
+            pathlib.Path("virtual-packages.yaml"),
         ]
         for c in candidates:
             if c.exists():
diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py
index 5568ac19e..43c5bc5dd 100644
--- a/conda_lock/virtual_package.py
+++ b/conda_lock/virtual_package.py
@@ -58,6 +58,7 @@ def __init__(self, base_dir: pathlib.Path):
         }
         self.all_repodata: Dict[str, dict] = {}
         self.hash: Optional[str] = None
+        self.old_env_vars: Dict[str, str] = {}
 
     @property
     def channel_url(self):
@@ -99,6 +100,26 @@ def write(self) -> None:
             logger.debug(filename)
         logger.debug("repo: %s", self.channel_url)
 
+    def __enter__(self):
+        """Ensure that if glibc etc is set by the overrides we force the conda solver overrride variables"""
+        env_vars_to_clear = set()
+        for package in self.packages_by_subdir:
+            if package.name.startswith("__"):
+                upper_name = package.name.lstrip("_").upper()
+                env_vars_to_clear.add(f"CONDA_OVERRIDE_{upper_name}")
+
+        for e in env_vars_to_clear:
+            self.old_env_vars[e] = os.environ.get(e)
+            os.environ[e] = ""
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Clear out old vars"""
+        for k, v in self.old_env_vars.items():
+            if v is None:
+                del os.environ[k]
+            else:
+                os.environ[k] = v
+
 
 def _init_fake_repodata() -> FakeRepoData:
     import shutil
@@ -169,6 +190,7 @@ def virtual_package_repo_from_specification(
 
     with virtual_package_spec_file.open("r") as fp:
         data = yaml.safe_load(fp)
+    logging.debug("Virtual package spec: %s", data)
 
     spec = VirtualPackageSpec.parse_obj(data)
 
diff --git a/tests/test-cuda/conda-linux-64.lock b/tests/test-cuda/conda-linux-64.lock
index 3e4241654..cdf1403a1 100644
--- a/tests/test-cuda/conda-linux-64.lock
+++ b/tests/test-cuda/conda-linux-64.lock
@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 1fbf01fc1830a11428199dab0c1c0f63db3468919e3434f4672cced029aea2cb
+# input_hash: db786fb3fdc60ae4e1723bb20dffa0cd73fc952e4955e64c6f6530e6573b16d9
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
 https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.1.0-h56837e0_8.tar.bz2#930957b6bff66cfd539ada080c5ca3e8
diff --git a/tests/test-cuda/conda-linux-64.lock.yml b/tests/test-cuda/conda-linux-64.lock.yml
new file mode 100644
index 000000000..298c04b0a
--- /dev/null
+++ b/tests/test-cuda/conda-linux-64.lock.yml
@@ -0,0 +1,15 @@
+# Generated by conda-lock.
+# platform: linux-64
+# input_hash: 1fbf01fc1830a11428199dab0c1c0f63db3468919e3434f4672cced029aea2cb
+
+channels:
+  - conda-forge
+dependencies:
+  - __glibc=2.17=0
+  - _libgcc_mutex=0.1=conda_forge
+  - libstdcxx-ng=11.1.0=h56837e0_8
+  - libgomp=11.1.0=hc902ee8_8
+  - _openmp_mutex=4.5=1_gnu
+  - libgcc-ng=11.1.0=hc902ee8_8
+  - cudatoolkit=11.0.3=h15472ef_8
+  - cudnn=8.0.5.39=ha5ca753_1
diff --git a/tests/test-cuda/environment.yml b/tests/test-cuda/environment.yml
deleted file mode 100644
index e56b319dc..000000000
--- a/tests/test-cuda/environment.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-name: cuda
-channels:
-  - conda-forge
-dependencies:
-  - cudatoolkit =11.0
-  - cudnn =8.0
\ No newline at end of file
diff --git a/tests/test-cuda/virtual-packages-old-glibc.yaml b/tests/test-cuda/virtual-packages-old-glibc.yaml
new file mode 100644
index 000000000..45724e2cd
--- /dev/null
+++ b/tests/test-cuda/virtual-packages-old-glibc.yaml
@@ -0,0 +1,4 @@
+subdirs:
+  linux-64:
+    packages:
+      __glibc: 2.11
\ No newline at end of file
diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py
index 3303ef973..81c486c30 100644
--- a/tests/test_conda_lock.py
+++ b/tests/test_conda_lock.py
@@ -37,6 +37,9 @@
 )
 
 
+TEST_DIR = pathlib.Path(__file__).parent
+
+
 @pytest.fixture(autouse=True)
 def logging_setup(caplog):
     caplog.set_level(logging.DEBUG)
@@ -44,12 +47,12 @@ def logging_setup(caplog):
 
 @pytest.fixture
 def gdal_environment():
-    return pathlib.Path(__file__).parent.joinpath("gdal").joinpath("environment.yml")
+    return TEST_DIR.joinpath("gdal").joinpath("environment.yml")
 
 
 @pytest.fixture
 def zlib_environment():
-    return pathlib.Path(__file__).parent.joinpath("zlib").joinpath("environment.yml")
+    return TEST_DIR.joinpath("zlib").joinpath("environment.yml")
 
 
 @pytest.fixture
@@ -63,21 +66,17 @@ def input_hash_zlib_environment():
 
 @pytest.fixture
 def meta_yaml_environment():
-    return pathlib.Path(__file__).parent.joinpath("test-recipe").joinpath("meta.yaml")
+    return TEST_DIR.joinpath("test-recipe").joinpath("meta.yaml")
 
 
 @pytest.fixture
 def poetry_pyproject_toml():
-    return (
-        pathlib.Path(__file__).parent.joinpath("test-poetry").joinpath("pyproject.toml")
-    )
+    return TEST_DIR.joinpath("test-poetry").joinpath("pyproject.toml")
 
 
 @pytest.fixture
 def flit_pyproject_toml():
-    return (
-        pathlib.Path(__file__).parent.joinpath("test-flit").joinpath("pyproject.toml")
-    )
+    return TEST_DIR.joinpath("test-flit").joinpath("pyproject.toml")
 
 
 @pytest.fixture(
@@ -469,19 +468,65 @@ def auth_():
     "stripped_lockfile,lockfile_with_auth",
     tuple(
         (
-            _read_file(
-                pathlib.Path(__file__)
-                .parent.joinpath("test-stripped-lockfile")
-                .joinpath(f"{filename}.lock")
-            ),
-            _read_file(
-                pathlib.Path(__file__)
-                .parent.joinpath("test-lockfile-with-auth")
-                .joinpath(f"{filename}.lock")
-            ),
+            _read_file(TEST_DIR / "test-stripped-lockfile" / f"{filename}.lock"),
+            _read_file(TEST_DIR / "test-lockfile-with-auth" / f"{filename}.lock"),
         )
         for filename in ("test",)
     ),
 )
 def test__add_auth_to_lockfile(stripped_lockfile, lockfile_with_auth, auth):
     assert _add_auth_to_lockfile(stripped_lockfile, auth) == lockfile_with_auth
+
+
+@pytest.mark.parametrize("kind", ["explicit", "env"])
+def test_virtual_packages(conda_exe, monkeypatch, kind):
+    test_dir = TEST_DIR.joinpath("test-cuda")
+    monkeypatch.chdir(test_dir)
+
+    if is_micromamba(conda_exe):
+        monkeypatch.setenv("CONDA_FLAGS", "-v")
+    if kind == "env" and not conda_supports_env(conda_exe):
+        pytest.skip(
+            f"Standalone conda @ '{conda_exe}' does not support materializing from environment files."
+        )
+
+    platform = "linux-64"
+
+    from click.testing import CliRunner, Result
+
+    runner = CliRunner(mix_stderr=False)
+    result = runner.invoke(
+        main,
+        [
+            "lock",
+            "--conda",
+            conda_exe,
+            "-p",
+            platform,
+            "-k",
+            kind,
+        ],
+    )
+
+    assert result.exit_code == 0
+
+    runner = CliRunner(mix_stderr=False)
+    result = runner.invoke(
+        main,
+        [
+            "lock",
+            "--conda",
+            conda_exe,
+            "-p",
+            platform,
+            "-k",
+            kind,
+            "--virtual-package-spec",
+            test_dir / "virtual-packages-old-glibc.yaml",
+        ],
+    )
+
+    # micromamba doesn't respect the CONDA_OVERRIDE_XXX="" env vars appropriately so it will include the
+    # system virtual packages regardless of whether they should be present.  Skip this check in that case
+    if not is_micromamba(conda_exe):
+        assert result.exit_code != 0

From 74c4c80e0280d8f76d63bad615c976a651ca9cf4 Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Thu, 9 Sep 2021 10:38:04 -0400
Subject: [PATCH 4/7] Fix test

---
 tests/test_conda_lock.py | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py
index 81c486c30..7e33175ae 100644
--- a/tests/test_conda_lock.py
+++ b/tests/test_conda_lock.py
@@ -200,22 +200,27 @@ def test_run_lock_with_input_hash_check(
 )
 def test_poetry_version_parsing_constraints(package, version, url_pattern):
     _conda_exe = determine_conda_executable("conda", mamba=False, micromamba=False)
-    spec = LockSpecification(
-        specs=[to_match_spec(package, poetry_version_to_conda_version(version))],
-        channels=["conda-forge"],
-        platform="linux-64",
-    )
-    lockfile_contents = create_lockfile_from_spec(
-        conda=_conda_exe,
-        spec=spec,
-        kind="explicit",
-    )
+    from conda_lock.virtual_package import default_virtual_package_repodata
 
-    for line in lockfile_contents:
-        if url_pattern in line:
-            break
-    else:
-        raise ValueError(f"could not find {package} {version}")
+    vpr = default_virtual_package_repodata()
+    with vpr:
+        spec = LockSpecification(
+            specs=[to_match_spec(package, poetry_version_to_conda_version(version))],
+            channels=["conda-forge"],
+            platform="linux-64",
+            virtual_package_repo=vpr,
+        )
+        lockfile_contents = create_lockfile_from_spec(
+            conda=_conda_exe,
+            spec=spec,
+            kind="explicit",
+        )
+
+        for line in lockfile_contents:
+            if url_pattern in line:
+                break
+        else:
+            raise ValueError(f"could not find {package} {version}")
 
 
 def test_aggregate_lock_specs():

From dbd5ad26632eb68ded09417393aa7812e8130aab Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Thu, 9 Sep 2021 11:07:44 -0400
Subject: [PATCH 5/7] Add test for input hash stability

---
 conda_lock/conda_lock.py                |  3 ++-
 tests/test-cuda/conda-linux-64.lock     |  2 +-
 tests/test-cuda/conda-linux-64.lock.yml |  1 -
 tests/test_conda_lock.py                | 20 ++++++++++++++++++++
 4 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py
index bc333ab0e..eb713a70a 100644
--- a/conda_lock/conda_lock.py
+++ b/conda_lock/conda_lock.py
@@ -555,7 +555,8 @@ def create_lockfile_from_spec(
                 *(
                     f'  - {pkg["name"]}={pkg["version"]}={pkg["build_string"]}'
                     for pkg in link_actions
-                    # TODO: exclude injected virtual packages
+                    # exclude virtual packages
+                    if not pkg["name"].startswith("__")
                 ),
             ]
         )
diff --git a/tests/test-cuda/conda-linux-64.lock b/tests/test-cuda/conda-linux-64.lock
index cdf1403a1..3e4241654 100644
--- a/tests/test-cuda/conda-linux-64.lock
+++ b/tests/test-cuda/conda-linux-64.lock
@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: db786fb3fdc60ae4e1723bb20dffa0cd73fc952e4955e64c6f6530e6573b16d9
+# input_hash: 1fbf01fc1830a11428199dab0c1c0f63db3468919e3434f4672cced029aea2cb
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
 https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.1.0-h56837e0_8.tar.bz2#930957b6bff66cfd539ada080c5ca3e8
diff --git a/tests/test-cuda/conda-linux-64.lock.yml b/tests/test-cuda/conda-linux-64.lock.yml
index 298c04b0a..e2371d893 100644
--- a/tests/test-cuda/conda-linux-64.lock.yml
+++ b/tests/test-cuda/conda-linux-64.lock.yml
@@ -5,7 +5,6 @@
 channels:
   - conda-forge
 dependencies:
-  - __glibc=2.17=0
   - _libgcc_mutex=0.1=conda_forge
   - libstdcxx-ng=11.1.0=h56837e0_8
   - libgomp=11.1.0=hc902ee8_8
diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py
index 7e33175ae..17b689483 100644
--- a/tests/test_conda_lock.py
+++ b/tests/test_conda_lock.py
@@ -535,3 +535,23 @@ def test_virtual_packages(conda_exe, monkeypatch, kind):
     # system virtual packages regardless of whether they should be present.  Skip this check in that case
     if not is_micromamba(conda_exe):
         assert result.exit_code != 0
+
+
+def test_virtual_package_input_hash_stability():
+    from conda_lock.virtual_package import (
+        default_virtual_package_repodata,
+        virtual_package_repo_from_specification,
+    )
+
+    test_dir = TEST_DIR.joinpath("test-cuda")
+    spec = test_dir / "virtual-packages-old-glibc.yaml"
+
+    vpr = virtual_package_repo_from_specification(spec)
+    spec = LockSpecification([], [], "linux-64", vpr)
+    expected = "ee2d9f11360510d22c6378f7a6e2ad9da251ae08c9ecf4757b3246021d82eb21"
+    assert spec.input_hash() == expected
+
+    vpr = default_virtual_package_repodata()
+    spec = LockSpecification([], [], "linux-64", vpr)
+    expected = "1e9efb84a8b0899ade9ea7319c4f10df7e0c6d28c5f306f6035cbf296d4e460e"
+    assert spec.input_hash() == expected

From f3ae074f6efaebf0cc0b0290ccbd971aafc680aa Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Thu, 9 Sep 2021 12:03:45 -0400
Subject: [PATCH 6/7] Rework virtual package hash to filter to only used
 platforms

---
 conda_lock/src_parser/__init__.py       |  6 ++-
 conda_lock/virtual_package.py           | 51 +++++++++++++++++++++----
 tests/test-cuda/conda-linux-64.lock     |  2 +-
 tests/test-cuda/conda-linux-64.lock.yml |  2 +-
 tests/test_conda_lock.py                | 31 +++++++++++----
 5 files changed, 74 insertions(+), 18 deletions(-)

diff --git a/conda_lock/src_parser/__init__.py b/conda_lock/src_parser/__init__.py
index f4741c59c..ebf4b4bf3 100644
--- a/conda_lock/src_parser/__init__.py
+++ b/conda_lock/src_parser/__init__.py
@@ -26,7 +26,11 @@ def input_hash(self) -> str:
             "specs": sorted(self.specs),
         }
         if self.virtual_package_repo is not None:
-            data["virtual_package_hash"] = self.virtual_package_repo.all_repodata
+            vpr_data = self.virtual_package_repo.all_repodata
+            data["virtual_package_hash"] = {
+                "noarch": vpr_data.get("noarch", {}),
+                self.platform: vpr_data.get(self.platform, {}),
+            }
 
         env_spec = json.dumps(data, sort_keys=True)
         return hashlib.sha256(env_spec.encode("utf-8")).hexdigest()
diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py
index 43c5bc5dd..bdd555571 100644
--- a/conda_lock/virtual_package.py
+++ b/conda_lock/virtual_package.py
@@ -17,9 +17,9 @@
 
 
 class FakePackage(BaseModel):
-    class Config:
-        """pydantic config."""
+    """A minimal representation of the required metadata for a conda package"""
 
+    class Config:
         allow_mutation = False
         frozen = True
 
@@ -147,13 +147,48 @@ def clean():
 
 
 def default_virtual_package_repodata() -> FakeRepoData:
+    """Define a reasonable modern set of virtual packages that should be safe enough to assume"""
     repodata = _init_fake_repodata()
-    fake_packages = [
-        FakePackage(name="__glibc", version="2.17"),
-        FakePackage(name="__cuda", version="11.4"),
-    ]
-    for pkg in fake_packages:
-        repodata.add_package(pkg)
+
+    unix_virtual = FakePackage(name="__unix", version="0")
+    repodata.add_package(
+        unix_virtual,
+        subdirs=["linux-aarch64", "linux-ppc64le", "linux-64", "osx-64", "osx-arm64"],
+    )
+
+    linux_virtual = FakePackage(name="__linux", version="5.10")
+    repodata.add_package(
+        linux_virtual, subdirs=["linux-aarch64", "linux-ppc64le", "linux-64"]
+    )
+
+    win_virtual = FakePackage(name="__win", version="0")
+    repodata.add_package(win_virtual, subdirs=["win-64"])
+
+    archspec_x86 = FakePackage(name="__archspec", version="1", build_string="x86_64")
+    repodata.add_package(archspec_x86, subdirs=["win-64", "linux-64", "osx-64"])
+
+    archspec_arm64 = FakePackage(name="__archspec", version="1", build_string="arm64")
+    repodata.add_package(archspec_arm64, subdirs=["osx-arm64"])
+
+    archspec_aarch64 = FakePackage(
+        name="__archspec", version="1", build_string="aarch64"
+    )
+    repodata.add_package(archspec_aarch64, subdirs=["linux-aarch64"])
+
+    archspec_ppc64le = FakePackage(
+        name="__archspec", version="1", build_string="ppc64le"
+    )
+    repodata.add_package(archspec_ppc64le, subdirs=["linux-ppc64le"])
+
+    glibc_virtual = FakePackage(name="__glibc", version="2.17")
+    repodata.add_package(
+        glibc_virtual, subdirs=["linux-aarch64", "linux-ppc64le", "linux-64"]
+    )
+
+    cuda_virtual = FakePackage(name="__cuda", version="11.4")
+    repodata.add_package(
+        cuda_virtual, subdirs=["linux-aarch64", "linux-ppc64le", "linux-64", "win-64"]
+    )
 
     for osx_ver in OSX_VERSIONS_X86:
         package = FakePackage(name="__osx", version=osx_ver)
diff --git a/tests/test-cuda/conda-linux-64.lock b/tests/test-cuda/conda-linux-64.lock
index 3e4241654..757a08414 100644
--- a/tests/test-cuda/conda-linux-64.lock
+++ b/tests/test-cuda/conda-linux-64.lock
@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 1fbf01fc1830a11428199dab0c1c0f63db3468919e3434f4672cced029aea2cb
+# input_hash: 8a6c4dd216afbbf86fc03e0b589a382ca1ea01ab894ed4c1b595a3a1066327fe
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
 https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.1.0-h56837e0_8.tar.bz2#930957b6bff66cfd539ada080c5ca3e8
diff --git a/tests/test-cuda/conda-linux-64.lock.yml b/tests/test-cuda/conda-linux-64.lock.yml
index e2371d893..1baeb1e0d 100644
--- a/tests/test-cuda/conda-linux-64.lock.yml
+++ b/tests/test-cuda/conda-linux-64.lock.yml
@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 1fbf01fc1830a11428199dab0c1c0f63db3468919e3434f4672cced029aea2cb
+# input_hash: 8a6c4dd216afbbf86fc03e0b589a382ca1ea01ab894ed4c1b595a3a1066327fe
 
 channels:
   - conda-forge
diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py
index 17b689483..475eb8bee 100644
--- a/tests/test_conda_lock.py
+++ b/tests/test_conda_lock.py
@@ -538,20 +538,37 @@ def test_virtual_packages(conda_exe, monkeypatch, kind):
 
 
 def test_virtual_package_input_hash_stability():
-    from conda_lock.virtual_package import (
-        default_virtual_package_repodata,
-        virtual_package_repo_from_specification,
-    )
+    from conda_lock.virtual_package import virtual_package_repo_from_specification
 
     test_dir = TEST_DIR.joinpath("test-cuda")
     spec = test_dir / "virtual-packages-old-glibc.yaml"
 
     vpr = virtual_package_repo_from_specification(spec)
     spec = LockSpecification([], [], "linux-64", vpr)
-    expected = "ee2d9f11360510d22c6378f7a6e2ad9da251ae08c9ecf4757b3246021d82eb21"
+    expected = "e8e6f657016351e26bef54e35091b6fcc76b266e1f136a8fa1f2f493d62d6dd6"
     assert spec.input_hash() == expected
 
+
+def _param(platform, hash):
+    return pytest.param(platform, hash, id=platform)
+
+
+@pytest.mark.parametrize(
+    ["platform", "expected"],
+    [
+        # fmt: off
+        _param("linux-64", "ed70aec6681f127c0bf2118c556c9e078afdab69b254b4e5aee12fdc8d7420b5"),
+        _param("linux-aarch64", "b30f28e2ad39531888479a67ac82c56c7fef041503f98eeb8b3cbaaa7a855ed9"),
+        _param("linux-ppc64le", "5b2235e1138500de742a291e3f0f26d68c61e6a6d4debadea106f4814055a28d"),
+        _param("osx-64", "b995edf1fe0718d3810b5cca77e235fa0e8c689179a79731bdc799418020bd3e"),
+        _param("osx-arm64", "e0a6f743325833c93d440e1dab0165afdf1b7d623740803b6cedc19f05618d73"),
+        _param("win-64", "3fe95154e8d7b99fa6326e025fb7d7ce44e4ae8253ac71e8f5b2acec50091c9e"),
+        # fmt: on
+    ],
+)
+def test_default_virtual_package_input_hash_stability(platform, expected):
+    from conda_lock.virtual_package import default_virtual_package_repodata
+
     vpr = default_virtual_package_repodata()
-    spec = LockSpecification([], [], "linux-64", vpr)
-    expected = "1e9efb84a8b0899ade9ea7319c4f10df7e0c6d28c5f306f6035cbf296d4e460e"
+    spec = LockSpecification([], [], platform, vpr)
     assert spec.input_hash() == expected

From af6c8c555bf4fb53f25773ca571b9706fb206faa Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Thu, 9 Sep 2021 12:21:26 -0400
Subject: [PATCH 7/7] Add readme

---
 README.md | 42 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 39363c599..5b918c283 100644
--- a/README.md
+++ b/README.md
@@ -82,7 +82,7 @@ conda-lock --no-dev-dependencies -f ./recipe/meta.yaml
 Under some situation you may want to run conda lock in some kind of automated way (eg as a precommit) and want to not
 need to regenerate the lockfiles if the underlying input specification for that particular lock as not changed.
 
-```
+```bash
 conda-lock --check-input-hash -p linux-64
 ```
 
@@ -106,10 +106,48 @@ In order to `conda-lock install` a lock file with its basic auth credentials str
 
 You can provide the authentication either as string through `--auth` or as a filepath through `--auth-file`.
 
-```
+```bash
 conda-lock install --auth-file auth.json conda-linux-64.lock
 ```
 
+### --virtual-package-spec
+
+Conda makes use of [virtual packages](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html) that are available at
+runtime to gate dependency on system features.  Due to these not generally existing on your local execution platform conda-lock will inject
+them into the solution environment with a reasonable guess at what a default system configuration should be.
+
+If you want to override which virtual packages are injected you can create a file like
+
+```yaml
+# virtual-packages.yml
+subdirs:
+  linux-64:
+    packages:
+      __glibc: 2.17
+      __cuda: 11.4
+  win-64:
+    packages:
+      __cuda: 11.4
+```
+
+conda-lock will automatically use a `virtual-packages.yml` it finds in the the current working directory.  Alternatively one can be specified
+explicitly via the flag.
+
+```bash
+conda lock --virtual-package-spec virtual-packages-cuda.yml -p linux-64
+```
+
+#### Input hash stability
+
+Virtual packages take part in the input hash so if you build an environment with a different set of virtual packages the input hash will change.
+Additionally the default set of virtual packages may be augmented in future versions of conda-lock.  If you desire very stable input hashes
+we recommend creating a `virtual-packages.yml` file to lock down the virtual packages considered.
+
+#### ⚠️ in conjunction with micromamba
+
+Micromamba does not presently support some of the overrides to remove all discovered virtual packages, consequently the set of virtual packages
+available at solve time may be larger than those specified in your specification.
+
 ## Supported file sources
 
 Conda lock supports more than just [environment.yml][envyaml] specifications!