WIP: implement gitlab !reference

scuba/config.py

@@ -56,6 +56,14 @@ class OverrideStr(str, OverrideMixin):
     pass
 
 
+class Reference(list):
+    """
+    Represents a `!reference` tag value that needs to be parsed after yaml is loaded
+    """
+
+    pass
+
+
 # http://stackoverflow.com/a/9577670
 class Loader(yaml.SafeLoader):
     _root: Path  # directory containing the loaded document
@@ -127,6 +135,64 @@ def from_yaml(self, node: yaml.nodes.Node) -> Any:
             raise yaml.YAMLError(f"Key {key!r} not found in {filename}")
         return cur
 
+    def from_gitlab(self, node: yaml.nodes.Node) -> Any:
+        """
+        Implementes a !from_yaml constructor with the following syntax:
+            !from_gitlab filename key
+
+        Arguments:
+            filename:   Filename of external YAML document from which to load,
+                        relative to the current YAML file.
+            key:        Key from external YAML document to return,
+                        using a dot-separated syntax for nested keys.
+
+        Examples:
+           `!from_gitlab external.yml pop`
+
+            `!from_gitlab external.yml foo.bar.pop`
+
+            `!from_gitlab "another file.yml" "foo bar.snap crackle.pop"`
+        """
+        # Load the content from the node, as a scalar
+        assert isinstance(node, yaml.nodes.ScalarNode)
+        content = self.construct_scalar(node)
+        assert isinstance(content, str)
+
+        # Split on unquoted spaces
+        parts = shlex.split(content)
+        if len(parts) != 2:
+            raise yaml.YAMLError("Two arguments expected to !from_gitlab")
+        filename, key = parts
+
+        # path is relative to the current YAML document
+        path = self._root / filename
+
+        # Load the other YAML document
+        doc = self._cache.get(path)
+        if not doc:
+            with path.open("r") as f:
+                doc = yaml.load(f, GitlabLoader)
+                self._cache[path] = doc
+
+        # Retrieve the key
+        try:
+            cur = doc
+            # Use a negative look-behind to split the key on non-escaped '.' characters
+            for k in re.split(r"(?<!\\)\.", key):
+                cur = cur[
+                    k.replace("\\.", ".")
+                ]  # Be sure to replace any escaped '.' characters with *just* the '.'
+        except KeyError:
+            raise yaml.YAMLError(f"Key {key!r} not found in {filename}")
+
+        if isinstance(cur, Reference):
+            cur = _process_reference(doc, cur)
+
+        if isinstance(cur, list):
+            cur = _resolve_reference_list(cur, doc)
+
+        return cur
+
     def override(self, node: yaml.nodes.Node) -> OverrideMixin:
         """
         Implements !override constructor
@@ -158,6 +224,33 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin:
 
 Loader.add_constructor("!from_yaml", Loader.from_yaml)
 Loader.add_constructor("!override", Loader.override)
+Loader.add_constructor("!from_gitlab", Loader.from_gitlab)
+
+
+class GitlabLoader(Loader):
+    # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
+    # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8
+
+    def reference(self, node: yaml.nodes.Node) -> Reference:
+        """
+        Implements a !reference constructor with the following syntax:
+            !reference [comma, separated, path]
+
+        Examples:
+            !reference [job, image]
+
+            !reference [other_job, before_script]
+
+            !reference [last_job, variables]
+        """
+        # Load the content from the node, as a sequence
+        assert isinstance(node, yaml.nodes.SequenceNode)
+        key = self.construct_sequence(node)
+        assert isinstance(key, list)
+        return Reference(key)
+
+
+GitlabLoader.add_constructor("!reference", GitlabLoader.reference)
 
 
 def find_config() -> Tuple[Path, Path, ScubaConfig]:
@@ -218,7 +311,7 @@ def _expand_env_vars(in_str: str) -> str:
         ) from ve
 
 
-def _process_script_node(node: CfgNode, name: str) -> List[str]:
+def _process_script_node(node: CfgNode, name: str, hook: bool = False) -> List[str]:
     """Process a script-type node
 
     Args:
@@ -233,6 +326,10 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]:
         # The script is just the text itself
         return [node]
 
+    if not hook and isinstance(node, list):
+        # if we're coming from a reference, the script may be a list despite not being a dict
+        return node
+
     if isinstance(node, dict):
         # There must be a "script" key, which must be a list of strings
         script = node.get("script")
@@ -250,6 +347,51 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]:
     raise ConfigError(f"{name}: must be string or dict")
 
 
+def _process_reference(doc: dict, key: Reference) -> Any:
+    """Process a reference tag
+
+    Args:
+      doc: a yaml document
+      key: the reference to be parsed
+
+    Returns:
+      the referenced value
+    """
+    # Retrieve the key
+    try:
+        cur = doc
+        for k in key:
+            cur = cur[k]
+    except KeyError:
+        raise yaml.YAMLError(f"Key {key!r} not found")
+
+    if isinstance(cur, list):
+        cur = _resolve_reference_list(cur, doc)
+
+    return cur
+
+
+def _resolve_reference_list(node_list: list, doc: dict):
+    """resolve nested references in a list node
+
+    Args:
+        list_node: a list node containing possibly containing references
+        doc: the current yaml doc
+
+    Returns:
+        a list with all references resolved
+    """
+    resolved_list = []
+    for node in node_list:
+        if isinstance(node, Reference):
+            # use += to concatenate list type
+            resolved_list += _process_reference(doc, node)
+        else:
+            # use append to concatenate other types
+            resolved_list.append(node)
+    return resolved_list
+
+
 def _process_environment(node: CfgNode, name: str) -> Environment:
     # Environment can be either a list of strings ("KEY=VALUE") or a mapping
     # Environment keys and values are always strings
@@ -594,7 +736,7 @@ def _load_hooks(self, data: CfgData) -> Dict[str, List[str]]:
         ):
             node = data.get("hooks", {}).get(name)
             if node:
-                hooks[name] = _process_script_node(node, name)
+                hooks[name] = _process_script_node(node, name, True)
         return hooks
 
     @property

tests/test_config.py

@@ -282,6 +282,183 @@ def test_load_config_safe_external(self) -> None:
         SCUBA_YML.write_text(f"image: !from_yaml {external_yml} danger")
         self.__test_load_config_safe(external_yml)
 
+    def test_load_config_from_gitlab_with_reference(self) -> None:
+        """load_config loads a config using !from_gitlab with !reference tag"""
+        GITLAB_YML.write_text(
+            """
+            .start:
+              here: dummian:8.2
+
+            now:
+              here: !reference [.start, here]
+            """
+        )
+        config = load_config(
+            config_text=f'image: !from_gitlab {GITLAB_YML} "now.here"\n'
+        )
+        assert config.image == "dummian:8.2"
+
+    def test_load_config_from_gitlab_reference_script_str(self) -> None:
+        """load a .gitlab-ci.yml with !reference in script"""
+        GITLAB_YML.write_text(
+            """
+            .setup:
+              script:
+                - do-something-really-important
+                - and-another-thing
+
+            build:
+              stage: build
+              script: !reference [.setup, script]
+            """
+        )
+        config = load_config(
+            config_text=f"""
+            aliases:
+              important: !from_gitlab {GITLAB_YML} build.script
+            """
+        )
+        assert config.aliases["important"].script == [
+            "do-something-really-important",
+            "and-another-thing",
+        ]
+
+    def test_load_config_from_gitlab_reference_script_list(self) -> None:
+        """load a .gitlab-ci.yml with !reference in script"""
+        GITLAB_YML.write_text(
+            """
+            .setup:
+              script:
+                - do-something-really-important
+
+            build:
+              stage: build
+              script:
+                - !reference [.setup, script]
+                - depends-on-important-stuff
+            """
+        )
+        config = load_config(
+            config_text=f"""
+            image: bosybux
+            aliases:
+              important: !from_gitlab {GITLAB_YML} build.script
+            """
+        )
+        assert config.aliases["important"].script == [
+            "do-something-really-important",
+            "depends-on-important-stuff",
+        ]
+
+    def test_load_config_from_gitlab_nested_reference(self) -> None:
+        """load a .gitlab-ci.yml with !reference in script"""
+        GITLAB_YML.write_text(
+            """
+            .initial:
+              script:
+                - the-most-important-thing
+
+            .setup:
+              script:
+                - !reference [.initial, script]
+                - do-something-really-important
+
+            build:
+              stage: build
+              script:
+                - !reference [.setup, script]
+                - depends-on-important-stuff
+            """
+        )
+        config = load_config(
+            config_text=f"""
+            image: bosybux
+            aliases:
+              important: !from_gitlab {GITLAB_YML} build.script
+            """
+        )
+        assert config.aliases["important"].script == [
+            "the-most-important-thing",
+            "do-something-really-important",
+            "depends-on-important-stuff",
+        ]
+
+    def test_load_config_from_gitlab_double_nested_reference(self) -> None:
+        """load a .gitlab-ci.yml with !reference in script"""
+        GITLAB_YML.write_text(
+            """
+            .preinit:
+              script:
+                - the-utmost-importance
+
+            .initial:
+              script:
+                - !reference [.preinit, script]
+                - the-most-important-thing
+
+            .setup:
+              script:
+                - !reference [.initial, script]
+                - do-something-really-important
+
+            build:
+              stage: build
+              script:
+                - !reference [.setup, script]
+                - depends-on-important-stuff
+            """
+        )
+        config = load_config(
+            config_text=f"""
+            image: bosybux
+            aliases:
+              important: !from_gitlab {GITLAB_YML} build.script
+            """
+        )
+        assert config.aliases["important"].script == [
+            "the-utmost-importance",
+            "the-most-important-thing",
+            "do-something-really-important",
+            "depends-on-important-stuff",
+        ]
+
+    def test_load_config_from_gitlab_with_include(self) -> None:
+        """
+        load_config loads a config using !from_gitlab with !reference tag while ignoring the include and extends
+        TODO: #200 implement other gitlab-specific yaml
+        """
+
+        GITLAB_YML.write_text(
+            f"""
+            include: dummy.yml
+
+            .base:
+              extends: .real_base
+              image: dummian:12
+              script:
+                - so something
+
+            other:
+              image: !reference [.base, image]
+            """
+        )
+        config = load_config(
+            config_text=f'image: !from_gitlab {GITLAB_YML} "other.image"\n',
+        )
+        assert config.image == "dummian:12"
+
+    def test_load_config_from_gitlab_with_bad_reference(self) -> None:
+        """load_config loads a config using !from_gitlab with !reference tag"""
+        GITLAB_YML.write_text(
+            """
+            now:
+              here: !reference [.start, here]
+            """
+        )
+        invalid_config(
+            config_text=f'image: !from_gitlab {GITLAB_YML} "now.here"\n',
+        )
+
 
 class TestConfigHooks(ConfigTest):
     def test_hooks_mixed(self) -> None: