diff --git a/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py b/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py index e142b7d..a122f54 100644 --- a/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py +++ b/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py @@ -1,16 +1,12 @@ # pyright: strict, reportTypeCommentUsage=false, reportMissingTypeStubs=false -import json import os -import subprocess -import sys import tempfile -import uuid from itertools import chain from typing import Dict, List, Optional, Set, Tuple, cast +from urllib.parse import urlparse from metaflow.debug import debug -from metaflow.metaflow_config import CONDA_LOCAL_PATH from metaflow._vendor.packaging.requirements import Requirement @@ -27,7 +23,6 @@ from ..utils import ( CondaException, - WithDir, arch_id, channel_or_url, parse_explicit_url_conda, @@ -35,6 +30,7 @@ pypi_tags_from_arch, ) from . import Resolver +from .pip_resolver import _FAKE_WHEEL class CondaLockResolver(Resolver): @@ -197,8 +193,13 @@ def resolve( for pypi_name, info in pypi_dep_lines.items(): if "url" in info: toml_lines.append( - '"%s" = {url = "%s", %s source="pypi"}\n' - % (pypi_name, info["url"], info["url_extras"]) + '"%s" = {%s = "%s", %s source="pypi"}\n' + % ( + pypi_name, + "git" if info["url"].startswith("git+") else "url", + info["url"], + info["url_extras"], + ) ) else: toml_lines.append( @@ -293,6 +294,34 @@ def resolve( raise CondaException( "Unexpected package specification line: %s" % l ) + debug.conda_exec("Got PYPI package %s" % l) + if components[4].startswith("git+"): + # This is a GIT URL so we have to build the package + # See pip_resolver.py for vcs_info on an explanation + # on how we deal with git packages + # Looks like gi+https://@ + base_build_url = components[4] + parse = urlparse(base_build_url) + clean_path, clean_commit = parse.path.split("@") + clean_url = parse.scheme[4:] + parse.netloc + clean_path + base_pkg_url = "%s/%s" % (clean_url, clean_commit) + # TODO: Do we need to handle subdirectories + cache_base_url = ( + PypiCachePackage.make_partial_cache_url( + base_pkg_url, is_real_url=False + ) + ) + packages_to_build[cache_base_url] = PackageToBuild( + base_build_url, + PypiPackageSpecification( + _FAKE_WHEEL, + base_pkg_url, + is_real_url=False, + url_format=".whl", + ), + ) + continue + # Non-GIT URL parse_result = parse_explicit_url_pypi(components[4]) if parse_result.url_format != ".whl": cache_base_url = ( @@ -370,27 +399,24 @@ def resolve( supported_tags = pypi_tags_from_arch( python_version, architecture, glibc_version ) - if self._conda.storage: - built_pypi_packages, builder_envs = build_pypi_packages( - self._conda, - self._conda.storage, - python_version, - packages_to_build, - builder_envs, - build_dir, - architecture, - supported_tags, - sources.get("pypi", []), - ) - packages.extend(built_pypi_packages) - else: - # Here it was just URLs so we are good - packages.extend( - [ - cast(PackageSpecification, v.spec) - for v in packages_to_build.values() - ] + if not self._conda.storage: + raise CondaException( + "Cannot create a relocatable environment as it depends on " + "local files or non wheels and no storage backend is defined: %s" + % ", ".join([v.url for v in packages_to_build.values()]) ) + built_pypi_packages, builder_envs = build_pypi_packages( + self._conda, + self._conda.storage, + python_version, + packages_to_build, + builder_envs, + build_dir, + architecture, + supported_tags, + sources.get("pypi", []), + ) + packages.extend(built_pypi_packages) return ( ResolvedEnvironment( deps, diff --git a/metaflow_extensions/netflix_ext/plugins/conda/utils.py b/metaflow_extensions/netflix_ext/plugins/conda/utils.py index 828331c..477a5e7 100644 --- a/metaflow_extensions/netflix_ext/plugins/conda/utils.py +++ b/metaflow_extensions/netflix_ext/plugins/conda/utils.py @@ -362,7 +362,12 @@ def parse_explicit_url_pypi(url: str) -> ParseExplicitResult: filename = None url_format = None - url_clean, url_hash = url.rsplit("#", 1) + splits = url.rsplit("#", 1) + if len(splits) > 1: + url_clean, url_hash = splits + else: + url_clean = url + url_hash = None if url_hash: if not url_hash.startswith("sha256="): raise CondaException("URL '%s' has a SHA type which is not supported" % url)