feat(actions): add docs propagation action for columns

.github/workflows/datahub-actions-docker.yml

@@ -55,16 +55,18 @@ jobs:
         uses: actions/checkout@v3
       - name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: docker/metadata-action@v5
         with:
           images: |
             acryldata/datahub-actions
-          tag-custom: ${{ needs.setup.outputs.tag }}
-          tag-custom-only: true
+          tags: |
+            type=raw,value=${{ needs.setup.outputs.tag }}
+          # tag-custom: ${{ needs.setup.outputs.tag }}
+          # tag-custom-only: true
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -78,8 +80,7 @@ jobs:
           tags: ${{ steps.docker_meta.outputs.tags }}
           push: ${{ needs.setup.outputs.publish == 'true' }}
           target: final
-          build-args:
-            "GEM_FURY_TOKEN=${{ secrets.GEMFURY_PULL_TOKEN }}"
+          build-args: 'GEM_FURY_TOKEN=${{ secrets.GEMFURY_PULL_TOKEN }}'
   slim_image:
     name: Build & Push Image to DockerHub (slim)
     runs-on: ubuntu-latest
@@ -89,13 +90,13 @@ jobs:
       - name: Check out the repo (slim)
         uses: actions/checkout@v3
       - name: Docker meta (slim)
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        id: docker_meta_slim
+        uses: docker/metadata-action@v5
         with:
           images: |
-            acryldata/datahub-actions
-          tag-custom: ${{ needs.setup.outputs.tag }}
-          tag-custom-only: true
+            acryldata/datahub-actions-slim
+          tags: |
+            type=raw,value=${{ needs.setup.outputs.tag }}
       - name: Set up QEMU (slim)
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx (slim)
@@ -105,14 +106,6 @@ jobs:
         with:
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
-      - name: Docker meta (slim)
-        id: docker_meta_slim
-        uses: crazy-max/ghaction-docker-meta@v1
-        with:
-          images: |
-            acryldata/datahub-actions-slim
-          tag-custom: ${{ needs.setup.outputs.tag }}
-          tag-custom-only: true
       - name: Build & Push Image (slim)
         uses: docker/build-push-action@v6
         with:
@@ -151,7 +144,7 @@ jobs:
   #         output: 'trivy-results.sarif'
   #         severity: 'CRITICAL,HIGH'
   #         ignore-unfixed: true
-  #         vuln-type: "os,library"          
+  #         vuln-type: "os,library"
   #     - name: Upload Trivy scan results to GitHub Security tab
   #       uses: github/codeql-action/upload-sarif@v2
   #       with:
@@ -161,12 +154,12 @@ jobs:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
       actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
-    name: "[Monitoring] Scan slim action images for vulnerabilities"
+    name: '[Monitoring] Scan slim action images for vulnerabilities'
     runs-on: ubuntu-latest
     needs: [setup, slim_image]
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Download image (slim)
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
@@ -183,8 +176,8 @@ jobs:
           output: 'trivy-results.sarif'
           severity: 'CRITICAL,HIGH'
           ignore-unfixed: true
-          vuln-type: "os,library"
+          vuln-type: 'os,library'
       - name: Upload Trivy scan results to GitHub Security tab (slim)
         uses: github/codeql-action/upload-sarif@v2
         with:
-          sarif_file: 'trivy-results.sarif'
+          sarif_file: 'trivy-results.sarif'

datahub-actions/setup.py

@@ -30,7 +30,7 @@ def get_long_description():
     return description
 
 
-acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION") or "0.12.1.5"
+acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION") or "0.13.3.6rc1"
 
 base_requirements = {
     f"acryl-datahub[datahub-kafka]>={acryl_datahub_min_version}",
@@ -86,6 +86,7 @@ def get_long_description():
     "snowflake_tag_propagation": {
         f"acryl-datahub[snowflake]>={acryl_datahub_min_version}"
     },
+    "doc_propagation": set(),
     # Transformer Plugins (None yet)
 }
 
@@ -138,6 +139,7 @@ def get_long_description():
             "tag_propagation",
             "term_propagation",
             "snowflake_tag_propagation",
+            "doc_propagation",
         ]
         for dependency in plugins[plugin]
     ),
@@ -158,6 +160,7 @@ def get_long_description():
             "tag_propagation",
             "term_propagation",
             "snowflake_tag_propagation",
+            "doc_propagation",
         ]
         for dependency in plugins[plugin]
     ),
@@ -173,6 +176,7 @@ def get_long_description():
         "tag_propagation = datahub_actions.plugin.action.tag.tag_propagation_action:TagPropagationAction",
         "term_propagation = datahub_actions.plugin.action.term.term_propagation_action:TermPropagationAction",
         "snowflake_tag_propagation = datahub_actions.plugin.action.snowflake.tag_propagator:SnowflakeTagPropagatorAction",
+        "doc_propagation = datahub_actions.plugin.action.propagation.docs.propagation_action:DocPropagationAction",
     ],
     "datahub_actions.transformer.plugins": [],
     "datahub_actions.source.plugins": [],

datahub-actions/src/datahub_actions/api/action_graph.py

@@ -186,6 +186,16 @@ def get_downstreams(self, entity_urn: str) -> List[str]:
             return entities
         return []
 
+    def get_upstreams(self, entity_urn: str) -> List[str]:
+        url_frag = f"/relationships?direction=OUTGOING&types=List(DownstreamOf)&urn={urllib.parse.quote(entity_urn)}"
+        url = f"{self.graph._gms_server}{url_frag}"
+        response = self.graph._get_generic(url)
+        if response["count"] > 0:
+            relnships = response["relationships"]
+            entities = [x["entity"] for x in relnships]
+            return entities
+        return []
+
     def get_relationships(
         self, entity_urn: str, direction: str, relationship_types: List[str]
     ) -> List[str]:

datahub-actions/src/datahub_actions/plugin/action/mcl_utils.py

@@ -0,0 +1,56 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable
+
+from datahub.metadata.schema_classes import MetadataChangeLogClass
+
+from datahub_actions.event.event_envelope import EventEnvelope
+from datahub_actions.event.event_registry import METADATA_CHANGE_LOG_EVENT_V1_TYPE
+
+
+class MCLProcessor:
+    """
+    A utility class to register and process MetadataChangeLog events.
+    """
+
+    def __init__(self) -> None:
+        self.entity_aspect_processors: dict[str, dict[str, Callable]] = {}
+        pass
+
+    def is_mcl(self, event: EventEnvelope) -> bool:
+        return event.event_type is METADATA_CHANGE_LOG_EVENT_V1_TYPE
+
+    def register_processor(
+        self, entity_type: str, aspect: str, processor: Callable
+    ) -> None:
+        if entity_type not in self.entity_aspect_processors:
+            self.entity_aspect_processors[entity_type] = {}
+        self.entity_aspect_processors[entity_type][aspect] = processor
+
+    def process(self, event: EventEnvelope) -> Any:
+
+        if isinstance(event.event, MetadataChangeLogClass):
+            entity_type = event.event.entityType
+            aspect = event.event.aspectName
+            if (
+                entity_type in self.entity_aspect_processors
+                and aspect in self.entity_aspect_processors[entity_type]
+            ):
+                return self.entity_aspect_processors[entity_type][aspect](
+                    entity_urn=event.event.entityUrn,
+                    aspect_name=event.event.aspectName,
+                    aspect_value=event.event.aspect,
+                    previous_aspect_value=event.event.previousAspectValue,
+                )

datahub-actions/src/datahub_actions/plugin/action/propagation/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

datahub-actions/src/datahub_actions/plugin/action/propagation/docs/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.