Merge pull request #117 from SimeonStoykovQC/disable-validate-platform

.github/workflows/pre-commit.yml

@@ -8,6 +8,8 @@ on:
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
+    env:
+      FORCE_COLOR: "1"
     steps:
     - uses: actions/checkout@v1
     - uses: actions/setup-python@v2

.github/workflows/test.yml

@@ -8,6 +8,7 @@ jobs:
   test-windows:
     env:
       PYTHONUNBUFFERED: "1"
+      FORCE_COLOR: "1"
     runs-on: windows-latest
     strategy:
       fail-fast: false
@@ -23,7 +24,7 @@ jobs:
         run: |
           conda activate test
           conda install mamba pip pytest-cov pytest-xdist
-          python -m pip install ensureconda==1.2.1
+          python -m pip install "ensureconda>=1.3"
           python -m pip install -r requirements.txt
           python -m pip install -r requirements-dev.txt
 
@@ -39,7 +40,7 @@ jobs:
           pushd "${RUNNER_TEMP}"
           set TMPDIR="%RUNNER_TEMP%"
           dir
-          pytest --showlocals -vrsx --cov=conda_lock tests
+          pytest -n auto --showlocals -vrsx --cov=conda_lock tests
 
   test:
     runs-on: ${{ matrix.os }}
@@ -53,6 +54,7 @@ jobs:
         shell: bash -l {0}
     env:
       PYTHONUNBUFFERED: "1"
+      FORCE_COLOR: "1"
     steps:
       - uses: actions/checkout@v2
 
@@ -70,7 +72,7 @@ jobs:
           echo "${PATH}"
           which pip
           which python
-          python -m pip install ensureconda==1.2.1
+          python -m pip install "ensureconda>=1.3"
           python -m pip install -r requirements.txt
           python -m pip install -r requirements-dev.txt
 
@@ -91,7 +93,7 @@ jobs:
           ls -lah
           set -x
           which pytest
-          pytest --showlocals -vrsx --cov=conda_lock tests
+          pytest -n auto --showlocals -vrsx --cov=conda_lock tests
 
       - name: test-gdal
         shell: bash -l {0}

.gitignore

@@ -8,3 +8,5 @@
 build/
 dist/
 venv/
+test_install*.yml
+test_install*.lock

conda_lock/conda_lock.py

@@ -16,6 +16,7 @@
 import tempfile
 
 from contextlib import contextmanager
+from distutils.version import LooseVersion
 from functools import partial
 from itertools import chain
 from typing import (
@@ -36,6 +37,7 @@
 import pkg_resources
 
 from click_default_group import DefaultGroup
+from ensureconda.api import determine_micromamba_version
 
 from conda_lock.common import read_file, read_json, write_file
 from conda_lock.errors import PlatformValidationError
@@ -71,6 +73,7 @@
 
 
 CONDA_PKGS_DIRS = None
+MAMBA_ROOT_PREFIX = None
 DEFAULT_PLATFORMS = ["osx-64", "linux-64", "win-64"]
 DEFAULT_KINDS = ["explicit"]
 KIND_FILE_EXT = {
@@ -143,6 +146,31 @@ def conda_pkgs_dir():
         return CONDA_PKGS_DIRS
 
 
+def mamba_root_prefix():
+    """Legacy root prefix used by micromamba"""
+    global MAMBA_ROOT_PREFIX
+    if MAMBA_ROOT_PREFIX is None:
+        temp_dir = tempfile.TemporaryDirectory()
+        MAMBA_ROOT_PREFIX = temp_dir.name
+        atexit.register(temp_dir.cleanup)
+        os.environ["MAMBA_ROOT_PREFIX"] = MAMBA_ROOT_PREFIX
+        return MAMBA_ROOT_PREFIX
+    else:
+        return MAMBA_ROOT_PREFIX
+
+
+def reset_conda_pkgs_dir():
+    """Clear the fake conda packages directory.  This is used only by testing"""
+    global CONDA_PKGS_DIRS
+    global MAMBA_ROOT_PREFIX
+    CONDA_PKGS_DIRS = None
+    MAMBA_ROOT_PREFIX = None
+    if "CONDA_PKGS_DIRS" in os.environ:
+        del os.environ["CONDA_PKGS_DIRS"]
+    if "MAMBA_ROOT_PREFIX" in os.environ:
+        del os.environ["MAMBA_ROOT_PREFIX"]
+
+
 def conda_env_override(platform) -> Dict[str, str]:
     env = dict(os.environ)
     env.update(
@@ -277,8 +305,6 @@ def do_conda_install(conda: PathLike, prefix: str, name: str, file: str) -> None
     if conda_flags:
         args.extend(shlex.split(conda_flags))
 
-    logging.debug("$MAMBA_ROOT_PREFIX: %s", os.environ.get("MAMBA_ROOT_PREFIX"))
-
     with subprocess.Popen(
         args,
         stdout=subprocess.PIPE,
@@ -313,21 +339,25 @@ def search_for_md5s(
     """
 
     def matchspec(spec):
-        return (
-            f"{spec['name']}["
-            f"version={spec['version']},"
-            f"subdir={spec['platform']},"
-            f"channel={spec['channel']},"
-            f"build={spec['build_string']}"
-            "]"
-        )
+        try:
+            return (
+                f"{spec['name']}["
+                f"version={spec['version']},"
+                f"subdir={spec.get('platform', platform)},"
+                f"channel={spec['channel']},"
+                f"build={spec['build_string']}"
+                "]"
+            )
+        except Exception:
+            logger.error("Failed to build a matchspec for %s", spec)
+            raise
 
     found: Set[str] = set()
     logging.debug("Searching for package specs: \n%s", package_specs)
     packages: List[Tuple[str, str]] = [
         *[(d["name"], matchspec(d)) for d in package_specs],
         *[(d["name"], f"{d['name']}[url='{d['url_conda']}']") for d in package_specs],
-        *[(d["name"], f"{d['name']}[url='{d['url']}']") for d in package_specs],
+        *[(d["name"], f"{d['name']}[url='{d['url_tar_bz2']}']") for d in package_specs],
     ]
 
     for name, spec in packages:
@@ -542,7 +572,7 @@ def create_lockfile_from_spec(
         channels=[*spec.channels, virtual_package_channel],
         specs=spec.specs,
     )
-    logging.debug("dry_run_install:\n%s", dry_run_install)
+    logging.debug("dry_run_install:\nspec: %s\nresult: %s", spec, dry_run_install)
 
     lockfile_contents = [
         "# Generated by conda-lock.",
@@ -577,15 +607,22 @@ def create_lockfile_from_spec(
                 link[
                     "url_base"
                 ] = f"{link['base_url']}/{link['platform']}/{link['dist_name']}"
-            link["url"] = f"{link['url_base']}.tar.bz2"
+            link["url_tar_bz2"] = f"{link['url_base']}.tar.bz2"
             link["url_conda"] = f"{link['url_base']}.conda"
-        link_dists = {link["dist_name"] for link in link_actions}
 
-        fetch_actions = dry_run_install["actions"]["FETCH"]
+        link_dists = {link["dist_name"] for link in link_actions}
+        link_dists_with_md5_and_url = {
+            link["dist_name"]
+            for link in link_actions
+            if bool(link.get("url")) and bool(link.get("md5"))
+        }
 
+        fetch_actions = dry_run_install["actions"].get("FETCH", [])
         fetch_by_dist_name = {fn_to_dist_name(pkg["fn"]): pkg for pkg in fetch_actions}
 
-        non_fetch_packages = link_dists - set(fetch_by_dist_name)
+        non_fetch_packages = (
+            link_dists - set(fetch_by_dist_name) - link_dists_with_md5_and_url
+        )
         if len(non_fetch_packages) > 0:
             for search_res in search_for_md5s(
                 conda=conda,
@@ -599,16 +636,18 @@ def create_lockfile_from_spec(
                 fetch_by_dist_name[dist_name] = search_res
 
         for pkg in link_actions:
-            dist_name = (
-                fn_to_dist_name(pkg["fn"]) if is_micromamba(conda) else pkg["dist_name"]
-            )
-            url = fetch_by_dist_name[dist_name]["url"]
+            dist_name = pkg["dist_name"]
+            url = pkg.get("url")
+            if not url:
+                url = fetch_by_dist_name[dist_name]["url"]
             if url.startswith(virtual_package_channel):
                 continue
             if url.startswith(spec.virtual_package_repo.channel_url_posix):
                 continue
             try:
-                md5 = fetch_by_dist_name[dist_name]["md5"]
+                md5 = pkg.get("md5")
+                if not md5:
+                    md5 = fetch_by_dist_name[dist_name]["md5"]
             except KeyError:
                 logger.error("failed to determine md5 for %s", url)
                 raise
@@ -726,11 +765,9 @@ def determine_conda_executable(
 ):
     for candidate in _determine_conda_executable(conda_executable, mamba, micromamba):
         if candidate is not None:
-            if is_micromamba(candidate) and "MAMBA_ROOT_PREFIX" not in os.environ:
-                mamba_root_prefix = pathlib.Path(candidate).parent / "mamba_root"
-                mamba_root_prefix.mkdir(exist_ok=True, parents=True)
-                os.environ["MAMBA_ROOT_PREFIX"] = str(mamba_root_prefix)
-
+            if is_micromamba(candidate):
+                if determine_micromamba_version(candidate) < LooseVersion("0.17"):
+                    mamba_root_prefix()
             return candidate
     raise RuntimeError("Could not find conda (or compatible) executable")
 
@@ -1023,8 +1060,7 @@ def handle_exception(exc_type, exc_value, exc_traceback):
 )
 @click.option("--auth-file", help="Path to the authentication file.", default="")
 @click.option(
-    "--validate-platform",
-    is_flag=True,
+    "--validate-platform/--no-validate-platform",
     default=True,
     help="Whether the platform compatibility between your lockfile and the host system should be validated.",
 )
@@ -1058,7 +1094,7 @@ def install(
             do_validate_platform(lockfile)
         except PlatformValidationError as error:
             raise PlatformValidationError(
-                error.args[0] + " Disable validation with `--validate-platform=False`."
+                error.args[0] + " Disable validation with `--no-validate-platform`."
             )
     if auth:
         lockfile = read_file(lock_file)

conda_lock/src_parser/__init__.py

@@ -19,6 +19,9 @@ def __init__(
         self.platform = platform
         self.virtual_package_repo = virtual_package_repo
 
+    def __str__(self) -> str:
+        return f"LockSpecification(specs={self.specs}, channels={self.specs}, platform={self.platform})"
+
     def input_hash(self) -> str:
         data: dict = {
             "channels": self.channels,

conda_lock/src_parser/pyproject_toml.py

@@ -90,7 +90,7 @@ def parse_poetry_pyproject_toml(
         for depname, depattrs in deps.items():
             conda_dep_name = normalize_pypi_name(depname)
             optional_dep = False
-            if isinstance(depattrs, collections.Mapping):
+            if isinstance(depattrs, collections.abc.Mapping):
                 poetry_version_spec = depattrs["version"]
                 optional_dep = depattrs.get("optional", False)
                 # TODO: support additional features such as markers for things like sys_platform, platform_system

requirements.txt

@@ -2,7 +2,7 @@ pyyaml
 requests
 jinja2
 toml
-ensureconda>=1.1.0
+ensureconda>=1.3.0
 click
 click-default-group
 pydantic

setup.cfg

@@ -25,6 +25,7 @@ classifiers =
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
 
 [options]
 zip_safe = True
@@ -33,7 +34,7 @@ install_requires =
     requests >=2
     Jinja2
     toml
-    ensureconda >=1.1
+    ensureconda >=1.3
     click
     click-default-group
     pydantic >=1.8.1

tests/test_conda_lock.py

@@ -19,12 +19,12 @@
     _strip_auth_from_line,
     _strip_auth_from_lockfile,
     aggregate_lock_specs,
-    conda_env_override,
     create_lockfile_from_spec,
     determine_conda_executable,
     is_micromamba,
     main,
     parse_meta_yaml_file,
+    reset_conda_pkgs_dir,
     run_lock,
 )
 from conda_lock.src_parser import LockSpecification
@@ -45,6 +45,11 @@ def logging_setup(caplog):
     caplog.set_level(logging.DEBUG)
 
 
+@pytest.fixture
+def reset_global_conda_pkgs_dir():
+    reset_conda_pkgs_dir()
+
+
 @pytest.fixture
 def gdal_environment():
     return TEST_DIR.joinpath("gdal").joinpath("environment.yml")
@@ -309,7 +314,9 @@ def conda_supports_env(conda_exe):
 
 
 @pytest.mark.parametrize("kind", ["explicit", "env"])
-def test_install(kind, tmp_path, conda_exe, zlib_environment, monkeypatch, capsys):
+def test_install(
+    request, kind, tmp_path, conda_exe, zlib_environment, monkeypatch, capsys
+):
     if is_micromamba(conda_exe):
         monkeypatch.setenv("CONDA_FLAGS", "-v")
     if kind == "env" and not conda_supports_env(conda_exe):
@@ -320,8 +327,14 @@ def test_install(kind, tmp_path, conda_exe, zlib_environment, monkeypatch, capsy
     package = "zlib"
     platform = "linux-64"
 
-    lock_filename_template = "conda-{platform}-{dev-dependencies}.lock"
-    lock_filename = "conda-linux-64-true.lock" + (".yml" if kind == "env" else "")
+    lock_filename_template = (
+        request.node.name + "conda-{platform}-{dev-dependencies}.lock"
+    )
+    lock_filename = (
+        request.node.name
+        + "conda-linux-64-true.lock"
+        + (".yml" if kind == "env" else "")
+    )
     try:
         os.remove(lock_filename)
     except OSError:
@@ -370,10 +383,11 @@ def invoke_install(*extra_args):
     result = invoke_install()
     print(result.stdout, file=sys.stdout)
     print(result.stderr, file=sys.stderr)
-    logging.debug(
-        "lockfile contents: \n\n=======\n%s\n\n==========",
-        pathlib.Path(lock_filename).read_text(),
-    )
+    if pathlib.Path(lock_filename).exists:
+        logging.debug(
+            "lockfile contents: \n\n=======\n%s\n\n==========",
+            pathlib.Path(lock_filename).read_text(),
+        )
     if sys.platform.lower().startswith("linux"):
         assert result.exit_code == 0
         assert _check_package_installed(