diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 9aab7a1fd..7b37f878a 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -8,6 +8,8 @@ on: jobs: pre-commit: runs-on: ubuntu-latest + env: + FORCE_COLOR: "1" steps: - uses: actions/checkout@v1 - uses: actions/setup-python@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 880f4263d..b9b6d3e46 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,7 @@ jobs: test-windows: env: PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" runs-on: windows-latest strategy: fail-fast: false @@ -23,7 +24,7 @@ jobs: run: | conda activate test conda install mamba pip pytest-cov pytest-xdist - python -m pip install ensureconda==1.2.1 + python -m pip install "ensureconda>=1.3" python -m pip install -r requirements.txt python -m pip install -r requirements-dev.txt @@ -39,7 +40,7 @@ jobs: pushd "${RUNNER_TEMP}" set TMPDIR="%RUNNER_TEMP%" dir - pytest --showlocals -vrsx --cov=conda_lock tests + pytest -n auto --showlocals -vrsx --cov=conda_lock tests test: runs-on: ${{ matrix.os }} @@ -53,6 +54,7 @@ jobs: shell: bash -l {0} env: PYTHONUNBUFFERED: "1" + FORCE_COLOR: "1" steps: - uses: actions/checkout@v2 @@ -70,7 +72,7 @@ jobs: echo "${PATH}" which pip which python - python -m pip install ensureconda==1.2.1 + python -m pip install "ensureconda>=1.3" python -m pip install -r requirements.txt python -m pip install -r requirements-dev.txt @@ -91,7 +93,7 @@ jobs: ls -lah set -x which pytest - pytest --showlocals -vrsx --cov=conda_lock tests + pytest -n auto --showlocals -vrsx --cov=conda_lock tests - name: test-gdal shell: bash -l {0} diff --git a/.gitignore b/.gitignore index 4e32a8024..9e5a3eaea 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ build/ dist/ venv/ +test_install*.yml +test_install*.lock diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index d6b0ff670..9351b7d00 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -16,6 +16,7 @@ import tempfile from contextlib import contextmanager +from distutils.version import LooseVersion from functools import partial from itertools import chain from typing import ( @@ -36,6 +37,7 @@ import pkg_resources from click_default_group import DefaultGroup +from ensureconda.api import determine_micromamba_version from conda_lock.common import read_file, read_json, write_file from conda_lock.errors import PlatformValidationError @@ -71,6 +73,7 @@ CONDA_PKGS_DIRS = None +MAMBA_ROOT_PREFIX = None DEFAULT_PLATFORMS = ["osx-64", "linux-64", "win-64"] DEFAULT_KINDS = ["explicit"] KIND_FILE_EXT = { @@ -143,6 +146,31 @@ def conda_pkgs_dir(): return CONDA_PKGS_DIRS +def mamba_root_prefix(): + """Legacy root prefix used by micromamba""" + global MAMBA_ROOT_PREFIX + if MAMBA_ROOT_PREFIX is None: + temp_dir = tempfile.TemporaryDirectory() + MAMBA_ROOT_PREFIX = temp_dir.name + atexit.register(temp_dir.cleanup) + os.environ["MAMBA_ROOT_PREFIX"] = MAMBA_ROOT_PREFIX + return MAMBA_ROOT_PREFIX + else: + return MAMBA_ROOT_PREFIX + + +def reset_conda_pkgs_dir(): + """Clear the fake conda packages directory. This is used only by testing""" + global CONDA_PKGS_DIRS + global MAMBA_ROOT_PREFIX + CONDA_PKGS_DIRS = None + MAMBA_ROOT_PREFIX = None + if "CONDA_PKGS_DIRS" in os.environ: + del os.environ["CONDA_PKGS_DIRS"] + if "MAMBA_ROOT_PREFIX" in os.environ: + del os.environ["MAMBA_ROOT_PREFIX"] + + def conda_env_override(platform) -> Dict[str, str]: env = dict(os.environ) env.update( @@ -277,8 +305,6 @@ def do_conda_install(conda: PathLike, prefix: str, name: str, file: str) -> None if conda_flags: args.extend(shlex.split(conda_flags)) - logging.debug("$MAMBA_ROOT_PREFIX: %s", os.environ.get("MAMBA_ROOT_PREFIX")) - with subprocess.Popen( args, stdout=subprocess.PIPE, @@ -313,21 +339,25 @@ def search_for_md5s( """ def matchspec(spec): - return ( - f"{spec['name']}[" - f"version={spec['version']}," - f"subdir={spec['platform']}," - f"channel={spec['channel']}," - f"build={spec['build_string']}" - "]" - ) + try: + return ( + f"{spec['name']}[" + f"version={spec['version']}," + f"subdir={spec.get('platform', platform)}," + f"channel={spec['channel']}," + f"build={spec['build_string']}" + "]" + ) + except Exception: + logger.error("Failed to build a matchspec for %s", spec) + raise found: Set[str] = set() logging.debug("Searching for package specs: \n%s", package_specs) packages: List[Tuple[str, str]] = [ *[(d["name"], matchspec(d)) for d in package_specs], *[(d["name"], f"{d['name']}[url='{d['url_conda']}']") for d in package_specs], - *[(d["name"], f"{d['name']}[url='{d['url']}']") for d in package_specs], + *[(d["name"], f"{d['name']}[url='{d['url_tar_bz2']}']") for d in package_specs], ] for name, spec in packages: @@ -542,7 +572,7 @@ def create_lockfile_from_spec( channels=[*spec.channels, virtual_package_channel], specs=spec.specs, ) - logging.debug("dry_run_install:\n%s", dry_run_install) + logging.debug("dry_run_install:\nspec: %s\nresult: %s", spec, dry_run_install) lockfile_contents = [ "# Generated by conda-lock.", @@ -577,15 +607,22 @@ def create_lockfile_from_spec( link[ "url_base" ] = f"{link['base_url']}/{link['platform']}/{link['dist_name']}" - link["url"] = f"{link['url_base']}.tar.bz2" + link["url_tar_bz2"] = f"{link['url_base']}.tar.bz2" link["url_conda"] = f"{link['url_base']}.conda" - link_dists = {link["dist_name"] for link in link_actions} - fetch_actions = dry_run_install["actions"]["FETCH"] + link_dists = {link["dist_name"] for link in link_actions} + link_dists_with_md5_and_url = { + link["dist_name"] + for link in link_actions + if bool(link.get("url")) and bool(link.get("md5")) + } + fetch_actions = dry_run_install["actions"].get("FETCH", []) fetch_by_dist_name = {fn_to_dist_name(pkg["fn"]): pkg for pkg in fetch_actions} - non_fetch_packages = link_dists - set(fetch_by_dist_name) + non_fetch_packages = ( + link_dists - set(fetch_by_dist_name) - link_dists_with_md5_and_url + ) if len(non_fetch_packages) > 0: for search_res in search_for_md5s( conda=conda, @@ -599,16 +636,18 @@ def create_lockfile_from_spec( fetch_by_dist_name[dist_name] = search_res for pkg in link_actions: - dist_name = ( - fn_to_dist_name(pkg["fn"]) if is_micromamba(conda) else pkg["dist_name"] - ) - url = fetch_by_dist_name[dist_name]["url"] + dist_name = pkg["dist_name"] + url = pkg.get("url") + if not url: + url = fetch_by_dist_name[dist_name]["url"] if url.startswith(virtual_package_channel): continue if url.startswith(spec.virtual_package_repo.channel_url_posix): continue try: - md5 = fetch_by_dist_name[dist_name]["md5"] + md5 = pkg.get("md5") + if not md5: + md5 = fetch_by_dist_name[dist_name]["md5"] except KeyError: logger.error("failed to determine md5 for %s", url) raise @@ -726,11 +765,9 @@ def determine_conda_executable( ): for candidate in _determine_conda_executable(conda_executable, mamba, micromamba): if candidate is not None: - if is_micromamba(candidate) and "MAMBA_ROOT_PREFIX" not in os.environ: - mamba_root_prefix = pathlib.Path(candidate).parent / "mamba_root" - mamba_root_prefix.mkdir(exist_ok=True, parents=True) - os.environ["MAMBA_ROOT_PREFIX"] = str(mamba_root_prefix) - + if is_micromamba(candidate): + if determine_micromamba_version(candidate) < LooseVersion("0.17"): + mamba_root_prefix() return candidate raise RuntimeError("Could not find conda (or compatible) executable") @@ -1023,8 +1060,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): ) @click.option("--auth-file", help="Path to the authentication file.", default="") @click.option( - "--validate-platform", - is_flag=True, + "--validate-platform/--no-validate-platform", default=True, help="Whether the platform compatibility between your lockfile and the host system should be validated.", ) @@ -1058,7 +1094,7 @@ def install( do_validate_platform(lockfile) except PlatformValidationError as error: raise PlatformValidationError( - error.args[0] + " Disable validation with `--validate-platform=False`." + error.args[0] + " Disable validation with `--no-validate-platform`." ) if auth: lockfile = read_file(lock_file) diff --git a/conda_lock/src_parser/__init__.py b/conda_lock/src_parser/__init__.py index ebf4b4bf3..703805217 100644 --- a/conda_lock/src_parser/__init__.py +++ b/conda_lock/src_parser/__init__.py @@ -19,6 +19,9 @@ def __init__( self.platform = platform self.virtual_package_repo = virtual_package_repo + def __str__(self) -> str: + return f"LockSpecification(specs={self.specs}, channels={self.specs}, platform={self.platform})" + def input_hash(self) -> str: data: dict = { "channels": self.channels, diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index b2a63c8a0..744c8656e 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -90,7 +90,7 @@ def parse_poetry_pyproject_toml( for depname, depattrs in deps.items(): conda_dep_name = normalize_pypi_name(depname) optional_dep = False - if isinstance(depattrs, collections.Mapping): + if isinstance(depattrs, collections.abc.Mapping): poetry_version_spec = depattrs["version"] optional_dep = depattrs.get("optional", False) # TODO: support additional features such as markers for things like sys_platform, platform_system diff --git a/requirements.txt b/requirements.txt index ab405b168..790d0dc17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ pyyaml requests jinja2 toml -ensureconda>=1.1.0 +ensureconda>=1.3.0 click click-default-group pydantic \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 37303c74c..7af7d6569 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [options] zip_safe = True @@ -33,7 +34,7 @@ install_requires = requests >=2 Jinja2 toml - ensureconda >=1.1 + ensureconda >=1.3 click click-default-group pydantic >=1.8.1 diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 15dbe7a8a..709cb4072 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -19,12 +19,12 @@ _strip_auth_from_line, _strip_auth_from_lockfile, aggregate_lock_specs, - conda_env_override, create_lockfile_from_spec, determine_conda_executable, is_micromamba, main, parse_meta_yaml_file, + reset_conda_pkgs_dir, run_lock, ) from conda_lock.src_parser import LockSpecification @@ -45,6 +45,11 @@ def logging_setup(caplog): caplog.set_level(logging.DEBUG) +@pytest.fixture +def reset_global_conda_pkgs_dir(): + reset_conda_pkgs_dir() + + @pytest.fixture def gdal_environment(): return TEST_DIR.joinpath("gdal").joinpath("environment.yml") @@ -309,7 +314,9 @@ def conda_supports_env(conda_exe): @pytest.mark.parametrize("kind", ["explicit", "env"]) -def test_install(kind, tmp_path, conda_exe, zlib_environment, monkeypatch, capsys): +def test_install( + request, kind, tmp_path, conda_exe, zlib_environment, monkeypatch, capsys +): if is_micromamba(conda_exe): monkeypatch.setenv("CONDA_FLAGS", "-v") if kind == "env" and not conda_supports_env(conda_exe): @@ -320,8 +327,14 @@ def test_install(kind, tmp_path, conda_exe, zlib_environment, monkeypatch, capsy package = "zlib" platform = "linux-64" - lock_filename_template = "conda-{platform}-{dev-dependencies}.lock" - lock_filename = "conda-linux-64-true.lock" + (".yml" if kind == "env" else "") + lock_filename_template = ( + request.node.name + "conda-{platform}-{dev-dependencies}.lock" + ) + lock_filename = ( + request.node.name + + "conda-linux-64-true.lock" + + (".yml" if kind == "env" else "") + ) try: os.remove(lock_filename) except OSError: @@ -370,10 +383,11 @@ def invoke_install(*extra_args): result = invoke_install() print(result.stdout, file=sys.stdout) print(result.stderr, file=sys.stderr) - logging.debug( - "lockfile contents: \n\n=======\n%s\n\n==========", - pathlib.Path(lock_filename).read_text(), - ) + if pathlib.Path(lock_filename).exists: + logging.debug( + "lockfile contents: \n\n=======\n%s\n\n==========", + pathlib.Path(lock_filename).read_text(), + ) if sys.platform.lower().startswith("linux"): assert result.exit_code == 0 assert _check_package_installed(