Skip to content

Commit

Permalink
Merge pull request #98 from matthewwardrop/add_support_for_outputting…
Browse files Browse the repository at this point in the history 
…_environment_files

Add support for outputting environment files.
  • Loading branch information
@mariusvniekerk
mariusvniekerk authored Jul 15, 2021
2 parents 272d94c + b8d8c7b commit dde2f64
Show file tree
Hide file tree
Showing 2 changed files with 166 additions and 74 deletions.
214 changes: 143 additions & 71 deletions conda_lock/conda_lock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,15 @@

CONDA_PKGS_DIRS = None
DEFAULT_PLATFORMS = ["osx-64", "linux-64", "win-64"]
DEFAULT_KINDS = ["explicit"]
KIND_FILE_EXT = {
"explicit": "",
"env": ".yml",
}
KIND_USE_TEXT = {
"explicit": "conda create --name YOURENV --file {lockfile}",
"env": "conda env create --name YOURENV --file {lockfile}",
}


def _extract_platform(line: str) -> Optional[str]:
Expand Down Expand Up @@ -212,12 +221,15 @@ def do_conda_install(conda: PathLike, prefix: str, name: str, file: str) -> None
if prefix and name:
raise ValueError("Provide either prefix, or name, but not both.")

kind = "env" if file.endswith(".yml") else "explicit"

args: MutableSequence[PathLike] = [
str(conda),
*(["env"] if kind == "env" else []),
"create",
"--file",
file,
"--yes",
*([] if kind == "env" else ["--yes"]),
]

if prefix:
Expand Down Expand Up @@ -320,6 +332,7 @@ def fn_to_dist_name(fn: str) -> str:
def make_lock_files(
conda: PathLike,
platforms: List[str],
kinds: List[str],
src_files: List[pathlib.Path],
include_dev_dependencies: bool = True,
channel_overrides: Optional[Sequence[str]] = None,
Expand Down Expand Up @@ -351,9 +364,17 @@ def make_lock_files(
file=sys.stderr,
)
sys.exit(1)
for kind, file_ext in KIND_FILE_EXT.items():
if file_ext and filename_template.endswith(file_ext):
print(
f"Filename template must not end with '{file_ext}', as this "
f"is reserved for '{kind}' lock files, in which case it is "
f"automatically added."
)
sys.exit(1)

for plat in platforms:
print(f"generating lockfile for {plat}", file=sys.stderr)
print(f"Generating lockfile(s) for {plat}...", file=sys.stderr)
lock_specs = parse_source_files(
src_files=src_files,
platform=plat,
Expand All @@ -366,48 +387,60 @@ def make_lock_files(
else:
channels = lock_spec.channels

lockfile_contents = create_lockfile_from_spec(
channels=channels, conda=conda, spec=lock_spec
)
for kind in kinds:
lockfile_contents = create_lockfile_from_spec(
channels=channels,
conda=conda,
spec=lock_spec,
kind=kind,
)

def sanitize_lockfile_line(line):
line = line.strip()
if line == "":
return "#"
if filename_template:
context = {
"platform": lock_spec.platform,
"dev-dependencies": str(include_dev_dependencies).lower(),
"spec-hash": lock_spec.env_hash(),
"version": pkg_resources.get_distribution("conda_lock").version,
"timestamp": datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"),
}

filename = filename_template.format(**context)
else:
return line
filename = f"conda-{lock_spec.platform}.lock"

if filename_template:
context = {
"platform": lock_spec.platform,
"dev-dependencies": str(include_dev_dependencies).lower(),
"spec-hash": lock_spec.env_hash(),
"version": pkg_resources.get_distribution("conda_lock").version,
"timestamp": datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"),
}
filename += KIND_FILE_EXT[kind]
with open(filename, "w") as fo:
fo.write("\n".join(lockfile_contents) + "\n")

filename = filename_template.format(**context)
else:
filename = f"conda-{lock_spec.platform}.lock"
with open(filename, "w") as fo:
fo.write(
"\n".join(sanitize_lockfile_line(ln) for ln in lockfile_contents) + "\n"
print(
f" - Install lock using {'(see warning below)' if kind == 'env' else ''}:",
KIND_USE_TEXT[kind].format(lockfile=filename),
file=sys.stderr,
)

print("To use the generated lock files create a new environment:", file=sys.stderr)
print("", file=sys.stderr)
print(
" conda create --name YOURENV --file conda-linux-64.lock", file=sys.stderr
)
print("", file=sys.stderr)
if "env" in kinds:
print(
"\nWARNING: Using environment lock files (*.yml) does NOT guarantee "
"that generated environments will be identical over time, since the "
"dependency resolver is re-run every time and changes in repository "
"metadata or resolver logic may cause variation. Conversely, since "
"the resolver is run every time, the resulting packages ARE "
"guaranteed to be seen by conda as being in a consistent state. This "
"makes them useful when updating existing environments.",
file=sys.stderr,
)


def is_micromamba(conda: PathLike) -> bool:
return str(conda).endswith("micromamba") or str(conda).endswith("micromamba.exe")


def create_lockfile_from_spec(
*, channels: Sequence[str], conda: PathLike, spec: LockSpecification
*,
channels: Sequence[str],
conda: PathLike,
spec: LockSpecification,
kind: str,
) -> List[str]:
dry_run_install = solve_specs_for_arch(
conda=conda,
Expand All @@ -416,53 +449,80 @@ def create_lockfile_from_spec(
specs=spec.specs,
)
logging.debug("dry_run_install:\n%s", dry_run_install)

lockfile_contents = [
"# Generated by conda-lock.",
f"# platform: {spec.platform}",
f"# env_hash: {spec.env_hash()}\n",
"@EXPLICIT\n",
]

link_actions = dry_run_install["actions"]["LINK"]
for link in link_actions:
if is_micromamba(conda):
link["url_base"] = fn_to_dist_name(link["url"])
link["dist_name"] = fn_to_dist_name(link["fn"])
else:
link[
"url_base"
] = f"{link['base_url']}/{link['platform']}/{link['dist_name']}"

link["url"] = f"{link['url_base']}.tar.bz2"
link["url_conda"] = f"{link['url_base']}.conda"
link_dists = {link["dist_name"] for link in link_actions}

fetch_actions = dry_run_install["actions"]["FETCH"]

fetch_by_dist_name = {fn_to_dist_name(pkg["fn"]): pkg for pkg in fetch_actions}

non_fetch_packages = link_dists - set(fetch_by_dist_name)
if len(non_fetch_packages) > 0:
for search_res in search_for_md5s(
conda=conda,
package_specs=[
x for x in link_actions if x["dist_name"] in non_fetch_packages
],
platform=spec.platform,
channels=channels,
):
dist_name = fn_to_dist_name(search_res["fn"])
fetch_by_dist_name[dist_name] = search_res

for pkg in link_actions:
dist_name = (
fn_to_dist_name(pkg["fn"]) if is_micromamba(conda) else pkg["dist_name"]
if kind == "env":
link_actions = dry_run_install["actions"]["LINK"]
lockfile_contents.extend(
[
"channels:",
*(f" - {channel}" for channel in channels),
"dependencies:",
*(
f' - {pkg["name"]}={pkg["version"]}={pkg["build_string"]}'
for pkg in link_actions
),
]
)
url = fetch_by_dist_name[dist_name]["url"]
md5 = fetch_by_dist_name[dist_name]["md5"]
lockfile_contents.append(f"{url}#{md5}")
elif kind == "explicit":
lockfile_contents.append("@EXPLICIT\n")

link_actions = dry_run_install["actions"]["LINK"]
for link in link_actions:
if is_micromamba(conda):
link["url_base"] = fn_to_dist_name(link["url"])
link["dist_name"] = fn_to_dist_name(link["fn"])
else:
link[
"url_base"
] = f"{link['base_url']}/{link['platform']}/{link['dist_name']}"

link["url"] = f"{link['url_base']}.tar.bz2"
link["url_conda"] = f"{link['url_base']}.conda"
link_dists = {link["dist_name"] for link in link_actions}

fetch_actions = dry_run_install["actions"]["FETCH"]

fetch_by_dist_name = {fn_to_dist_name(pkg["fn"]): pkg for pkg in fetch_actions}

non_fetch_packages = link_dists - set(fetch_by_dist_name)
if len(non_fetch_packages) > 0:
for search_res in search_for_md5s(
conda=conda,
package_specs=[
x for x in link_actions if x["dist_name"] in non_fetch_packages
],
platform=spec.platform,
channels=channels,
):
dist_name = fn_to_dist_name(search_res["fn"])
fetch_by_dist_name[dist_name] = search_res

for pkg in link_actions:
dist_name = (
fn_to_dist_name(pkg["fn"]) if is_micromamba(conda) else pkg["dist_name"]
)
url = fetch_by_dist_name[dist_name]["url"]
md5 = fetch_by_dist_name[dist_name]["md5"]
lockfile_contents.append(f"{url}#{md5}")

logging.debug("lockfile_contents:\n%s\n", lockfile_contents)
def sanitize_lockfile_line(line):
line = line.strip()
if line == "":
return "#"
else:
return line

lockfile_contents = [sanitize_lockfile_line(line) for line in lockfile_contents]
else:
raise ValueError(f"Unrecognised lock kind {kind}.")

logging.debug("lockfile_contents:\n%s\n", lockfile_contents)
return lockfile_contents


Expand Down Expand Up @@ -631,6 +691,7 @@ def run_lock(
include_dev_dependencies: bool = True,
channel_overrides: Optional[Sequence[str]] = None,
filename_template: Optional[str] = None,
kinds: Optional[List[str]] = None,
) -> None:
_conda_exe = determine_conda_executable(
conda_exe, mamba=mamba, micromamba=micromamba
Expand All @@ -642,6 +703,7 @@ def run_lock(
include_dev_dependencies=include_dev_dependencies,
channel_overrides=channel_overrides,
filename_template=filename_template,
kinds=kinds or DEFAULT_KINDS,
)


Expand Down Expand Up @@ -691,10 +753,18 @@ def main():
multiple=True,
help="path to a conda environment specification(s)",
)
@click.option(
"-k",
"--kind",
default=["explicit"],
type=str,
multiple=True,
help="Kind of lock file(s) to generate [should be one of 'explicit' or 'env'].",
)
@click.option(
"--filename-template",
default="conda-{platform}.lock",
help="Template for the lock file names. Must include {platform} token. For a full list and description of available tokens, see the command help text.",
help="Template for the lock file names. Filename must include {platform} token, and must not end in '.yml'. For a full list and description of available tokens, see the command help text.",
)
@click.option(
"--strip-auth",
Expand Down Expand Up @@ -726,6 +796,7 @@ def lock(
channel_overrides,
dev_dependencies,
files,
kind,
filename_template,
strip_auth,
log_level,
Expand Down Expand Up @@ -753,6 +824,7 @@ def lock(
micromamba=micromamba,
include_dev_dependencies=dev_dependencies,
channel_overrides=channel_overrides,
kinds=kind,
)
if strip_auth:
with tempfile.TemporaryDirectory() as tempdir:
Expand Down
26 changes: 23 additions & 3 deletions tests/test_conda_lock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,10 @@ def test_poetry_version_parsing_constraints(package, version, url_pattern):
platform="linux-64",
)
lockfile_contents = create_lockfile_from_spec(
conda=_conda_exe, channels=spec.channels, spec=spec
conda=_conda_exe,
channels=spec.channels,
spec=spec,
kind="explicit",
)

for line in lockfile_contents:
Expand Down Expand Up @@ -235,15 +238,30 @@ def _check_package_installed(package: str, prefix: str):
return True


def test_install(tmp_path, conda_exe, zlib_environment, monkeypatch):
def conda_supports_env(conda_exe):
try:
subprocess.check_call(
[conda_exe, "env"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
except subprocess.CalledProcessError:
return False
return True


@pytest.mark.parametrize("kind", ["explicit", "env"])
def test_install(kind, tmp_path, conda_exe, zlib_environment, monkeypatch):
if is_micromamba(conda_exe):
monkeypatch.setenv("CONDA_FLAGS", "-v")
if kind == "env" and not conda_supports_env(conda_exe):
pytest.skip(
f"Standalone conda @ '{conda_exe}' does not support materializing from environment files."
)

package = "zlib"
platform = "linux-64"

lock_filename_template = "conda-{platform}-{dev-dependencies}.lock"
lock_filename = "conda-linux-64-true.lock"
lock_filename = "conda-linux-64-true.lock" + (".yml" if kind == "env" else "")
try:
os.remove(lock_filename)
except OSError:
Expand All @@ -262,6 +280,8 @@ def test_install(tmp_path, conda_exe, zlib_environment, monkeypatch):
platform,
"-f",
zlib_environment,
"-k",
kind,
"--filename-template",
lock_filename_template,
],
Expand Down

0 comments on commit dde2f64

Please sign in to comment.