diff --git a/scuba/config.py b/scuba/config.py index ba4a507..24e73eb 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -56,6 +56,14 @@ class OverrideStr(str, OverrideMixin): pass +class Reference(list): + """ + Represents a `!reference` tag value that needs to be parsed after yaml is loaded + """ + + pass + + # http://stackoverflow.com/a/9577670 class Loader(yaml.SafeLoader): _root: Path # directory containing the loaded document @@ -127,6 +135,64 @@ def from_yaml(self, node: yaml.nodes.Node) -> Any: raise yaml.YAMLError(f"Key {key!r} not found in {filename}") return cur + def from_gitlab(self, node: yaml.nodes.Node) -> Any: + """ + Implementes a !from_yaml constructor with the following syntax: + !from_gitlab filename key + + Arguments: + filename: Filename of external YAML document from which to load, + relative to the current YAML file. + key: Key from external YAML document to return, + using a dot-separated syntax for nested keys. + + Examples: + `!from_gitlab external.yml pop` + + `!from_gitlab external.yml foo.bar.pop` + + `!from_gitlab "another file.yml" "foo bar.snap crackle.pop"` + """ + # Load the content from the node, as a scalar + assert isinstance(node, yaml.nodes.ScalarNode) + content = self.construct_scalar(node) + assert isinstance(content, str) + + # Split on unquoted spaces + parts = shlex.split(content) + if len(parts) != 2: + raise yaml.YAMLError("Two arguments expected to !from_gitlab") + filename, key = parts + + # path is relative to the current YAML document + path = self._root / filename + + # Load the other YAML document + doc = self._cache.get(path) + if not doc: + with path.open("r") as f: + doc = yaml.load(f, GitlabLoader) + self._cache[path] = doc + + # Retrieve the key + try: + cur = doc + # Use a negative look-behind to split the key on non-escaped '.' characters + for k in re.split(r"(? OverrideMixin: """ Implements !override constructor @@ -158,6 +224,33 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin: Loader.add_constructor("!from_yaml", Loader.from_yaml) Loader.add_constructor("!override", Loader.override) +Loader.add_constructor("!from_gitlab", Loader.from_gitlab) + + +class GitlabLoader(Loader): + # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags + # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8 + + def reference(self, node: yaml.nodes.Node) -> Reference: + """ + Implements a !reference constructor with the following syntax: + !reference [comma, separated, path] + + Examples: + !reference [job, image] + + !reference [other_job, before_script] + + !reference [last_job, variables] + """ + # Load the content from the node, as a sequence + assert isinstance(node, yaml.nodes.SequenceNode) + key = self.construct_sequence(node) + assert isinstance(key, list) + return Reference(key) + + +GitlabLoader.add_constructor("!reference", GitlabLoader.reference) def find_config() -> Tuple[Path, Path, ScubaConfig]: @@ -218,7 +311,7 @@ def _expand_env_vars(in_str: str) -> str: ) from ve -def _process_script_node(node: CfgNode, name: str) -> List[str]: +def _process_script_node(node: CfgNode, name: str, hook: bool = False) -> List[str]: """Process a script-type node Args: @@ -233,6 +326,10 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: # The script is just the text itself return [node] + if not hook and isinstance(node, list): + # if we're coming from a reference, the script may be a list despite not being a dict + return node + if isinstance(node, dict): # There must be a "script" key, which must be a list of strings script = node.get("script") @@ -250,6 +347,51 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: raise ConfigError(f"{name}: must be string or dict") +def _process_reference(doc: dict, key: Reference) -> Any: + """Process a reference tag + + Args: + doc: a yaml document + key: the reference to be parsed + + Returns: + the referenced value + """ + # Retrieve the key + try: + cur = doc + for k in key: + cur = cur[k] + except KeyError: + raise yaml.YAMLError(f"Key {key!r} not found") + + if isinstance(cur, list): + cur = _resolve_reference_list(cur, doc) + + return cur + + +def _resolve_reference_list(node_list: list, doc: dict): + """resolve nested references in a list node + + Args: + list_node: a list node containing possibly containing references + doc: the current yaml doc + + Returns: + a list with all references resolved + """ + resolved_list = [] + for node in node_list: + if isinstance(node, Reference): + # use += to concatenate list type + resolved_list += _process_reference(doc, node) + else: + # use append to concatenate other types + resolved_list.append(node) + return resolved_list + + def _process_environment(node: CfgNode, name: str) -> Environment: # Environment can be either a list of strings ("KEY=VALUE") or a mapping # Environment keys and values are always strings @@ -594,7 +736,7 @@ def _load_hooks(self, data: CfgData) -> Dict[str, List[str]]: ): node = data.get("hooks", {}).get(name) if node: - hooks[name] = _process_script_node(node, name) + hooks[name] = _process_script_node(node, name, True) return hooks @property diff --git a/tests/test_config.py b/tests/test_config.py index 95b255e..7897a5d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -282,6 +282,183 @@ def test_load_config_safe_external(self) -> None: SCUBA_YML.write_text(f"image: !from_yaml {external_yml} danger") self.__test_load_config_safe(external_yml) + def test_load_config_from_gitlab_with_reference(self) -> None: + """load_config loads a config using !from_gitlab with !reference tag""" + GITLAB_YML.write_text( + """ + .start: + here: dummian:8.2 + + now: + here: !reference [.start, here] + """ + ) + config = load_config( + config_text=f'image: !from_gitlab {GITLAB_YML} "now.here"\n' + ) + assert config.image == "dummian:8.2" + + def test_load_config_from_gitlab_reference_script_str(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .setup: + script: + - do-something-really-important + - and-another-thing + + build: + stage: build + script: !reference [.setup, script] + """ + ) + config = load_config( + config_text=f""" + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "do-something-really-important", + "and-another-thing", + ] + + def test_load_config_from_gitlab_reference_script_list(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .setup: + script: + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "do-something-really-important", + "depends-on-important-stuff", + ] + + def test_load_config_from_gitlab_nested_reference(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .initial: + script: + - the-most-important-thing + + .setup: + script: + - !reference [.initial, script] + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "the-most-important-thing", + "do-something-really-important", + "depends-on-important-stuff", + ] + + def test_load_config_from_gitlab_double_nested_reference(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .preinit: + script: + - the-utmost-importance + + .initial: + script: + - !reference [.preinit, script] + - the-most-important-thing + + .setup: + script: + - !reference [.initial, script] + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == [ + "the-utmost-importance", + "the-most-important-thing", + "do-something-really-important", + "depends-on-important-stuff", + ] + + def test_load_config_from_gitlab_with_include(self) -> None: + """ + load_config loads a config using !from_gitlab with !reference tag while ignoring the include and extends + TODO: #200 implement other gitlab-specific yaml + """ + + GITLAB_YML.write_text( + f""" + include: dummy.yml + + .base: + extends: .real_base + image: dummian:12 + script: + - so something + + other: + image: !reference [.base, image] + """ + ) + config = load_config( + config_text=f'image: !from_gitlab {GITLAB_YML} "other.image"\n', + ) + assert config.image == "dummian:12" + + def test_load_config_from_gitlab_with_bad_reference(self) -> None: + """load_config loads a config using !from_gitlab with !reference tag""" + GITLAB_YML.write_text( + """ + now: + here: !reference [.start, here] + """ + ) + invalid_config( + config_text=f'image: !from_gitlab {GITLAB_YML} "now.here"\n', + ) + class TestConfigHooks(ConfigTest): def test_hooks_mixed(self) -> None: