From 1a974808ab358aa08da8e8d7a0c18260b650bebb Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Tue, 26 Nov 2024 16:08:01 -0500 Subject: [PATCH 1/2] refactor recipe sync options --- CHANGELOG.md | 3 + percy/commands/recipe.py | 31 ++++-- percy/updater/grayskull_sync.py | 185 +++++++++++++++++--------------- pyproject.toml | 2 +- 4 files changed, 122 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b7d919..2e1f8ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog Note: version releases in the 0.x.y range may introduce breaking changes. +## 0.2.1 +- Bug fixes around percy recipe sync + ## 0.2.0 - Add command percy recipe sync - Download cbc when none is found. diff --git a/percy/commands/recipe.py b/percy/commands/recipe.py index d240fec..a6c371b 100644 --- a/percy/commands/recipe.py +++ b/percy/commands/recipe.py @@ -244,31 +244,40 @@ def patch( @recipe.command(short_help="Sync a recipe from pypi data") @click.pass_obj @click.option( - "--run_constrained", + "--no_run_constrained", type=bool, is_flag=True, show_default=True, - default=True, + default=False, + multiple=False, + help="do not add run_constrained", +) +@click.option( + "--no_bump", + type=bool, + is_flag=True, + show_default=True, + default=False, multiple=False, - help="add run_constrained", + help="do not bump build number if version is unchanged", ) @click.option( - "--bump", + "--no_linter", type=bool, is_flag=True, show_default=True, - default=True, + default=False, multiple=False, - help="bump build number if version is unchanged", + help="do not run conda lint --fix after updating", ) @click.option( - "--run_linter", + "--no_temp_files", type=bool, is_flag=True, show_default=True, - default=True, + default=False, multiple=False, - help="run conda lint --fix after updating", + help="do not keep intermediate files", ) @click.option( "--pypi_spec", @@ -276,10 +285,10 @@ def patch( multiple=False, help="pypi_package spec", ) -def sync(obj, pypi_spec, run_constrained, bump, run_linter): +def sync(obj, pypi_spec, no_run_constrained, no_bump, no_linter, no_temp_files): """ Sync a recipe from pypi data """ recipe_path = obj["recipe_path"] - grayskull_sync.sync(recipe_path, pypi_spec, run_constrained, bump, run_linter) + grayskull_sync.sync(recipe_path, pypi_spec, no_run_constrained, no_bump, no_linter, no_temp_files) diff --git a/percy/updater/grayskull_sync.py b/percy/updater/grayskull_sync.py index 4a3f180..6fc842f 100644 --- a/percy/updater/grayskull_sync.py +++ b/percy/updater/grayskull_sync.py @@ -10,6 +10,7 @@ """ import logging +import os import re import subprocess import tempfile @@ -19,11 +20,7 @@ from percy.render._renderer import RendererType -def gen_grayskull_recipe( - gs_file_path: Path, - package_spec: str, - with_run_constrained: str = True, -): +def gen_grayskull_recipe(gs_file_path: Path, package_spec: str, no_run_constrained: bool = True): """ Calls grayskull and format the resulting recipe for easy processing. @@ -31,7 +28,7 @@ def gen_grayskull_recipe( :param package_spec: Package spec accepted by grayskull pypi - :param with_run_constrained: Add run_constrained sections + :param no_run_constrained: Do not add run_constrained sections :returns: (raw_recipe, rendered_recipe, sections) """ @@ -39,7 +36,7 @@ def gen_grayskull_recipe( with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as tmp_file: # call grayskull cmd = "grayskull pypi" - if with_run_constrained: + if not no_run_constrained: cmd += " --extras-require-all" cmd += f" -o {tmp_file.name} {package_spec}" print(cmd) @@ -80,7 +77,14 @@ def gen_grayskull_recipe( return raw_recipe, rendered_recipe, sections -def sync(recipe_path: Path, package_spec: str | None, with_run_constrained: bool, bump: bool, run_linter: bool): +def sync( + recipe_path: Path, + package_spec: str | None, + no_run_constrained: bool, + no_bump: bool, + no_linter: bool, + no_temp_files: bool, +): """ Sync a recipe with content fetched from grayskull. @@ -88,11 +92,13 @@ def sync(recipe_path: Path, package_spec: str | None, with_run_constrained: bool :param package_spec: Package spec accepted by grayskull pypi (optional) - :param with_run_constrained: Add run_constrained sections + :param no_run_constrained: Do not add run_constrained sections - :param bump: If no version update, bump build number + :param no_bump: If no version update, do not bump build number - :param run_linter: Run linter + :param no_linter: Do not run linter + + :param no_temp_files: Do leave intermediate files """ @@ -116,82 +122,87 @@ def sync(recipe_path: Path, package_spec: str | None, with_run_constrained: bool package_spec = next(iter(rendered_recipe.packages.keys())) except Exception as error: # pylint: disable=broad-exception-caught logging.error("Failed to render existing recipe. %s", error) - - try: - # load grayskull recipe - gs_file_path = recipe_path.parent / "grayskull.yaml" - gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe( - gs_file_path, package_spec, with_run_constrained - ) - except Exception as error: # pylint: disable=broad-exception-caught - logging.error("Failed to load data from grayskull. %s", error) - - try: - # sync changes - grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") - local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") - if grayskull_version == local_version: - # no version change, bump build number - if bump: - build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) - raw_recipe.set_build_number(build_number) - else: - raw_recipe.set_version(grayskull_version) - raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) - raw_recipe.set_build_number("0") - - # build dep patch instructions - patch_instructions = [] - sep_map = { - ">=": "<", - ">": "<=", - "==": "!=", - "!=": "==", - "<=": ">", - "<": ">=", - } - skip_value = None - for section in sections: + else: + try: + # load grayskull recipe + gs_file_path = recipe_path.parent / "grayskull.yaml" + gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe( + gs_file_path, package_spec, no_run_constrained + ) + if no_temp_files: try: - for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): - pkg_spec = pkg_spec.replace("<{", "{{") - if pkg_spec.startswith("python "): - for sep, opp in sep_map.items(): - s = pkg_spec.split(sep) - if len(s) > 1: - skip_value = f"py{opp}{s[1].strip().replace('.','')}" - pkg_spec = "python" - break - - section_name = section - print(section, pkg_spec) - if section.startswith("run_constrained"): - if len(pkg_spec.split()) < 2: - continue - (section_name, extra) = section.rsplit("_", 1) - pkg_spec = f"{pkg_spec} # extra:{extra}" - - patch_instructions.append( - { - "op": "add", - "path": f"requirements/{section_name}", - "match": rf"{pkg_spec.split()[0]}( .*)?", - "value": [pkg_spec], - } - ) - except KeyError as e: - print(e) - continue - - if skip_value: - raw_recipe.update_py_skip(skip_value) - raw_recipe.patch(patch_instructions, False, False) + os.remove(gs_file_path) + except OSError: + pass + try: + # sync changes + grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") + local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") + if grayskull_version == local_version: + # no version change, bump build number + if not no_bump: + build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) + raw_recipe.set_build_number(build_number) + else: + raw_recipe.set_version(grayskull_version) + raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) + raw_recipe.set_build_number("0") + + # build dep patch instructions + patch_instructions = [] + sep_map = { + ">=": "<", + ">": "<=", + "==": "!=", + "!=": "==", + "<=": ">", + "<": ">=", + } + skip_value = None + for section in sections: + try: + for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): + pkg_spec = pkg_spec.replace("<{", "{{") + if pkg_spec.startswith("python "): + for sep, opp in sep_map.items(): + s = pkg_spec.split(sep) + if len(s) > 1: + skip_value = "".join(s[1].strip().split(".")[:2]) + skip_value = f"py{opp}{skip_value}" + pkg_spec = "python" + break + + section_name = section + print(section, pkg_spec) + if section.startswith("run_constrained"): + if len(pkg_spec.split()) < 2: + continue + (section_name, extra) = section.rsplit("_", 1) + pkg_spec = f"{pkg_spec} # extra:{extra}" + + patch_instructions.append( + { + "op": "add", + "path": f"requirements/{section_name}", + "match": rf"{pkg_spec.split()[0]}( .*)?", + "value": [pkg_spec], + } + ) + except KeyError as e: + print(e) + continue + + if skip_value: + raw_recipe.update_py_skip(skip_value) + raw_recipe.patch(patch_instructions, False, False) - try: - # run linter with autofix - if run_linter: - subprocess.call("conda lint . --fix", text=True, shell=True) + try: + # run linter with autofix + if not no_linter: + subprocess.call("conda lint . --fix", text=True, shell=True) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to lint synced recipe. %s", error) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to sync changes to recipe. %s", error) except Exception as error: # pylint: disable=broad-exception-caught - logging.error("Failed to lint synced recipe. %s", error) - except Exception as error: # pylint: disable=broad-exception-caught - logging.error("Failed to sync changes to recipe. %s", error) + logging.error("Failed to load data from grayskull. %s", error) diff --git a/pyproject.toml b/pyproject.toml index 4cfd325..6ad13a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ namespaces = false [project] name = "percy" -version = "0.2.0" +version = "0.2.1" authors = [ { name="Anaconda, Inc.", email="distribution_team@anaconda.com" }, ] From 2bfc676438e854de066b6494a0870c1d7243b38b Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Tue, 26 Nov 2024 17:15:42 -0500 Subject: [PATCH 2/2] refactor exception handling --- percy/updater/grayskull_sync.py | 167 ++++++++++++++++---------------- 1 file changed, 86 insertions(+), 81 deletions(-) diff --git a/percy/updater/grayskull_sync.py b/percy/updater/grayskull_sync.py index 6fc842f..053d75f 100644 --- a/percy/updater/grayskull_sync.py +++ b/percy/updater/grayskull_sync.py @@ -122,87 +122,92 @@ def sync( package_spec = next(iter(rendered_recipe.packages.keys())) except Exception as error: # pylint: disable=broad-exception-caught logging.error("Failed to render existing recipe. %s", error) - else: - try: - # load grayskull recipe - gs_file_path = recipe_path.parent / "grayskull.yaml" - gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe( - gs_file_path, package_spec, no_run_constrained - ) - if no_temp_files: - try: - os.remove(gs_file_path) - except OSError: - pass + return + + try: + # load grayskull recipe + gs_file_path = recipe_path.parent / "grayskull.yaml" + gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe( + gs_file_path, package_spec, no_run_constrained + ) + if no_temp_files: try: - # sync changes - grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") - local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") - if grayskull_version == local_version: - # no version change, bump build number - if not no_bump: - build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) - raw_recipe.set_build_number(build_number) - else: - raw_recipe.set_version(grayskull_version) - raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) - raw_recipe.set_build_number("0") - - # build dep patch instructions - patch_instructions = [] - sep_map = { - ">=": "<", - ">": "<=", - "==": "!=", - "!=": "==", - "<=": ">", - "<": ">=", - } - skip_value = None - for section in sections: - try: - for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): - pkg_spec = pkg_spec.replace("<{", "{{") - if pkg_spec.startswith("python "): - for sep, opp in sep_map.items(): - s = pkg_spec.split(sep) - if len(s) > 1: - skip_value = "".join(s[1].strip().split(".")[:2]) - skip_value = f"py{opp}{skip_value}" - pkg_spec = "python" - break - - section_name = section - print(section, pkg_spec) - if section.startswith("run_constrained"): - if len(pkg_spec.split()) < 2: - continue - (section_name, extra) = section.rsplit("_", 1) - pkg_spec = f"{pkg_spec} # extra:{extra}" - - patch_instructions.append( - { - "op": "add", - "path": f"requirements/{section_name}", - "match": rf"{pkg_spec.split()[0]}( .*)?", - "value": [pkg_spec], - } - ) - except KeyError as e: - print(e) - continue - - if skip_value: - raw_recipe.update_py_skip(skip_value) - raw_recipe.patch(patch_instructions, False, False) + os.remove(gs_file_path) + except OSError: + pass + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to load data from grayskull. %s", error) + return + try: + # sync changes + grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") + local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") + if grayskull_version == local_version: + # no version change, bump build number + if not no_bump: + build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) + raw_recipe.set_build_number(build_number) + else: + raw_recipe.set_version(grayskull_version) + raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) + raw_recipe.set_build_number("0") + + # build dep patch instructions + patch_instructions = [] + sep_map = { + ">=": "<", + ">": "<=", + "==": "!=", + "!=": "==", + "<=": ">", + "<": ">=", + } + skip_value = None + for section in sections: try: - # run linter with autofix - if not no_linter: - subprocess.call("conda lint . --fix", text=True, shell=True) - except Exception as error: # pylint: disable=broad-exception-caught - logging.error("Failed to lint synced recipe. %s", error) - except Exception as error: # pylint: disable=broad-exception-caught - logging.error("Failed to sync changes to recipe. %s", error) - except Exception as error: # pylint: disable=broad-exception-caught - logging.error("Failed to load data from grayskull. %s", error) + for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): + pkg_spec = pkg_spec.replace("<{", "{{") + if pkg_spec.startswith("python "): + for sep, opp in sep_map.items(): + s = pkg_spec.split(sep) + if len(s) > 1: + skip_value = "".join(s[1].strip().split(".")[:2]) + skip_value = f"py{opp}{skip_value}" + pkg_spec = "python" + break + + section_name = section + print(section, pkg_spec) + if section.startswith("run_constrained"): + if len(pkg_spec.split()) < 2: + continue + (section_name, extra) = section.rsplit("_", 1) + pkg_spec = f"{pkg_spec} # extra:{extra}" + + patch_instructions.append( + { + "op": "add", + "path": f"requirements/{section_name}", + "match": rf"{pkg_spec.split()[0]}( .*)?", + "value": [pkg_spec], + } + ) + except KeyError as e: + print(e) + continue + + if skip_value: + raw_recipe.update_py_skip(skip_value) + raw_recipe.patch(patch_instructions, False, False) + + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to sync changes to recipe. %s", error) + return + + try: + # run linter with autofix + if not no_linter: + subprocess.call("conda lint . --fix", text=True, shell=True) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to lint synced recipe. %s", error)