From 1b56a19b1bdee868e97d1a3fb3d0ab9c68f2173b Mon Sep 17 00:00:00 2001 From: funkecoder23 Date: Thu, 27 Jun 2024 12:42:43 -0400 Subject: [PATCH 1/3] add support for !from_gitlab and !reference --- scuba/config.py | 100 +++++++++++++++++++++++++++++++++++++++++++ tests/test_config.py | 28 ++++++++++++ 2 files changed, 128 insertions(+) diff --git a/scuba/config.py b/scuba/config.py index ba4a507..c59bd64 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -55,6 +55,11 @@ class OverrideList(list, OverrideMixin): class OverrideStr(str, OverrideMixin): pass +class Reference(list): + """ + Represents a `!reference` tag value that needs to be parsed after yaml is loaded + """ + pass # http://stackoverflow.com/a/9577670 class Loader(yaml.SafeLoader): @@ -127,6 +132,61 @@ def from_yaml(self, node: yaml.nodes.Node) -> Any: raise yaml.YAMLError(f"Key {key!r} not found in {filename}") return cur + def from_gitlab(self, node: yaml.nodes.Node) -> Any: + """ + Implementes a !from_yaml constructor with the following syntax: + !from_gitlab filename key + + Arguments: + filename: Filename of external YAML document from which to load, + relative to the current YAML file. + key: Key from external YAML document to return, + using a dot-separated syntax for nested keys. + + Examples: + `!from_gitlab external.yml pop` + + `!from_gitlab external.yml foo.bar.pop` + + `!from_gitlab "another file.yml" "foo bar.snap crackle.pop"` + """ + # Load the content from the node, as a scalar + assert isinstance(node, yaml.nodes.ScalarNode) + content = self.construct_scalar(node) + assert isinstance(content, str) + + # Split on unquoted spaces + parts = shlex.split(content) + if len(parts) != 2: + raise yaml.YAMLError("Two arguments expected to !from_gitlab") + filename, key = parts + + # path is relative to the current YAML document + path = self._root / filename + + # Load the other YAML document + doc = self._cache.get(path) + if not doc: + with path.open("r") as f: + doc = yaml.load(f, GitlabLoader) + self._cache[path] = doc + + # Retrieve the key + try: + cur = doc + # Use a negative look-behind to split the key on non-escaped '.' characters + for k in re.split(r"(? OverrideMixin: """ Implements !override constructor @@ -158,6 +218,46 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin: Loader.add_constructor("!from_yaml", Loader.from_yaml) Loader.add_constructor("!override", Loader.override) +Loader.add_constructor("!from_gitlab", Loader.from_gitlab) + +class GitlabLoader(Loader): + # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags + # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8 + + def reference(self, node: yaml.nodes.Node) -> Reference: + """ + Implements a !reference constructor with the following syntax: + !reference [comma, separated, path] + + Examples: + !reference [job, image] + + !reference [other_job, before_script] + + !reference [last_job, variables] + """ + # Load the content from the node, as a sequence + assert isinstance(node, yaml.nodes.SequenceNode) + key = self.construct_sequence(node) + assert isinstance(key, list) + return Reference(key) + + +def _process_reference(doc: dict, key: Reference) -> Any: + """ + Converts a reference (list of yaml keys) to its referenced value + """ + # Retrieve the key + try: + cur = doc + for k in key: + cur = cur[k] + except KeyError: + raise yaml.YAMLError(f"Key {key!r} not found") + return cur + +GitlabLoader.add_constructor("!reference", GitlabLoader.reference) + def find_config() -> Tuple[Path, Path, ScubaConfig]: diff --git a/tests/test_config.py b/tests/test_config.py index 95b255e..e816da3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -282,6 +282,34 @@ def test_load_config_safe_external(self) -> None: SCUBA_YML.write_text(f"image: !from_yaml {external_yml} danger") self.__test_load_config_safe(external_yml) + def test_load_config_from_gitlab_with_reference(self) -> None: + """load_config loads a config using !from_gitlab with !reference tag""" + GITLAB_YML.write_text( + """ + .start: + here: dummian:8.2 + + now: + here: !reference [.start, here] + """ + ) + config = load_config( + config_text=f'image: !from_gitlab {GITLAB_YML} "now.here"\n' + ) + assert config.image == "dummian:8.2" + + def test_load_config_from_gitlab_with_bad_reference(self) -> None: + """load_config loads a config using !from_gitlab with !reference tag""" + GITLAB_YML.write_text( + """ + now: + here: !reference [.start, here] + """ + ) + invalid_config( + config_text=f'image: !from_gitlab {GITLAB_YML} "now.here"\n', + ) + class TestConfigHooks(ConfigTest): def test_hooks_mixed(self) -> None: From 932c8bb29d20bb8d4d071b5d0add23e2b7864ae1 Mon Sep 17 00:00:00 2001 From: funkecoder23 Date: Thu, 27 Jun 2024 13:55:11 -0400 Subject: [PATCH 2/3] add support for reference in scripts --- scuba/config.py | 69 ++++++++++++++++++++++++++++++++++---------- tests/test_config.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 16 deletions(-) diff --git a/scuba/config.py b/scuba/config.py index c59bd64..735c613 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -55,12 +55,15 @@ class OverrideList(list, OverrideMixin): class OverrideStr(str, OverrideMixin): pass + class Reference(list): """ Represents a `!reference` tag value that needs to be parsed after yaml is loaded """ + pass + # http://stackoverflow.com/a/9577670 class Loader(yaml.SafeLoader): _root: Path # directory containing the loaded document @@ -181,10 +184,13 @@ def from_gitlab(self, node: yaml.nodes.Node) -> Any: ] # Be sure to replace any escaped '.' characters with *just* the '.' except KeyError: raise yaml.YAMLError(f"Key {key!r} not found in {filename}") - + if isinstance(cur, Reference): cur = _process_reference(doc, cur) + if isinstance(cur, list): + cur = _resolve_reference_list(cur, doc) + return cur def override(self, node: yaml.nodes.Node) -> OverrideMixin: @@ -220,10 +226,11 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin: Loader.add_constructor("!override", Loader.override) Loader.add_constructor("!from_gitlab", Loader.from_gitlab) + class GitlabLoader(Loader): # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8 - + def reference(self, node: yaml.nodes.Node) -> Reference: """ Implements a !reference constructor with the following syntax: @@ -242,24 +249,10 @@ def reference(self, node: yaml.nodes.Node) -> Reference: assert isinstance(key, list) return Reference(key) - -def _process_reference(doc: dict, key: Reference) -> Any: - """ - Converts a reference (list of yaml keys) to its referenced value - """ - # Retrieve the key - try: - cur = doc - for k in key: - cur = cur[k] - except KeyError: - raise yaml.YAMLError(f"Key {key!r} not found") - return cur GitlabLoader.add_constructor("!reference", GitlabLoader.reference) - def find_config() -> Tuple[Path, Path, ScubaConfig]: """Search up the directory hierarchy for .scuba.yml @@ -333,6 +326,9 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: # The script is just the text itself return [node] + if isinstance(node, list): + return node + if isinstance(node, dict): # There must be a "script" key, which must be a list of strings script = node.get("script") @@ -350,6 +346,47 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: raise ConfigError(f"{name}: must be string or dict") +def _process_reference(doc: dict, key: Reference) -> Any: + """Process a reference tag + + Args: + doc: a yaml document + key: the reference to be parsed + + Returns: + the referenced value + """ + # Retrieve the key + try: + cur = doc + for k in key: + cur = cur[k] + except KeyError: + raise yaml.YAMLError(f"Key {key!r} not found") + return cur + + +def _resolve_reference_list(node_list: list, doc: dict): + """resolve nested references in a list node + + Args: + list_node: a list node containing possibly containing references + doc: the current yaml doc + + Returns: + a list with all references resolved + """ + resolved_list = [] + for node in node_list: + if isinstance(node, Reference): + # use += to concatenate list type + resolved_list += _process_reference(doc, node) + else: + # use append to concatenate other types + resolved_list.append(node) + return resolved_list + + def _process_environment(node: CfgNode, name: str) -> Environment: # Environment can be either a list of strings ("KEY=VALUE") or a mapping # Environment keys and values are always strings diff --git a/tests/test_config.py b/tests/test_config.py index e816da3..801d4bd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -298,6 +298,57 @@ def test_load_config_from_gitlab_with_reference(self) -> None: ) assert config.image == "dummian:8.2" + def test_load_config_from_gitlab_reference_script_str(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .setup: + script: + - do-something-really-important + - and-another-thing + + build: + stage: build + script: !reference [.setup, script] + """ + ) + config = load_config( + config_text=f""" + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "do-something-really-important", + "and-another-thing", + ] + + def test_load_config_from_gitlab_reference_script_list(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .setup: + script: + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.image == "bosybux" + assert len(config.aliases) == 1 + assert config.aliases["important"].script == ["do-something-really-important", "depends-on-important-stuff"] + def test_load_config_from_gitlab_with_bad_reference(self) -> None: """load_config loads a config using !from_gitlab with !reference tag""" GITLAB_YML.write_text( From abb491ce8595005643201f87db3dd835933b5680 Mon Sep 17 00:00:00 2001 From: funkecoder23 Date: Thu, 27 Jun 2024 14:23:32 -0400 Subject: [PATCH 3/3] add (probably bad) way to handle nested references --- scuba/config.py | 11 +++-- tests/test_config.py | 104 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 109 insertions(+), 6 deletions(-) diff --git a/scuba/config.py b/scuba/config.py index 735c613..24e73eb 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -311,7 +311,7 @@ def _expand_env_vars(in_str: str) -> str: ) from ve -def _process_script_node(node: CfgNode, name: str) -> List[str]: +def _process_script_node(node: CfgNode, name: str, hook: bool = False) -> List[str]: """Process a script-type node Args: @@ -326,7 +326,8 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: # The script is just the text itself return [node] - if isinstance(node, list): + if not hook and isinstance(node, list): + # if we're coming from a reference, the script may be a list despite not being a dict return node if isinstance(node, dict): @@ -363,6 +364,10 @@ def _process_reference(doc: dict, key: Reference) -> Any: cur = cur[k] except KeyError: raise yaml.YAMLError(f"Key {key!r} not found") + + if isinstance(cur, list): + cur = _resolve_reference_list(cur, doc) + return cur @@ -731,7 +736,7 @@ def _load_hooks(self, data: CfgData) -> Dict[str, List[str]]: ): node = data.get("hooks", {}).get(name) if node: - hooks[name] = _process_script_node(node, name) + hooks[name] = _process_script_node(node, name, True) return hooks @property diff --git a/tests/test_config.py b/tests/test_config.py index 801d4bd..7897a5d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -345,9 +345,107 @@ def test_load_config_from_gitlab_reference_script_list(self) -> None: important: !from_gitlab {GITLAB_YML} build.script """ ) - assert config.image == "bosybux" - assert len(config.aliases) == 1 - assert config.aliases["important"].script == ["do-something-really-important", "depends-on-important-stuff"] + assert config.aliases["important"].script == [ + "do-something-really-important", + "depends-on-important-stuff", + ] + + def test_load_config_from_gitlab_nested_reference(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .initial: + script: + - the-most-important-thing + + .setup: + script: + - !reference [.initial, script] + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "the-most-important-thing", + "do-something-really-important", + "depends-on-important-stuff", + ] + + def test_load_config_from_gitlab_double_nested_reference(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .preinit: + script: + - the-utmost-importance + + .initial: + script: + - !reference [.preinit, script] + - the-most-important-thing + + .setup: + script: + - !reference [.initial, script] + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "the-utmost-importance", + "the-most-important-thing", + "do-something-really-important", + "depends-on-important-stuff", + ] + + def test_load_config_from_gitlab_with_include(self) -> None: + """ + load_config loads a config using !from_gitlab with !reference tag while ignoring the include and extends + TODO: #200 implement other gitlab-specific yaml + """ + + GITLAB_YML.write_text( + f""" + include: dummy.yml + + .base: + extends: .real_base + image: dummian:12 + script: + - so something + + other: + image: !reference [.base, image] + """ + ) + config = load_config( + config_text=f'image: !from_gitlab {GITLAB_YML} "other.image"\n', + ) + assert config.image == "dummian:12" def test_load_config_from_gitlab_with_bad_reference(self) -> None: """load_config loads a config using !from_gitlab with !reference tag"""