fix(model): fixes DashboardContainsDashboard relationship in DashboardInfo aspect

docs/how/updating-datahub.md

@@ -28,6 +28,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Other Notable Changes
 
+- #12433: Fixes the searchable annotations in the model supporting `Dashboard` to `Dashboard` lineage within the `DashboardInfo` aspect. Users of Sigma and PowerBI Apps ingestion may need to run a targeted [reindex](https://datahubproject.io/docs/how/restore-indices/) to update metadata on these edges.
+
 ## 0.15.0
 
 - OpenAPI Update: PIT Keep Alive parameter added to scroll endpoints. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set.

entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java

@@ -9,6 +9,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.Edge;
 import com.linkedin.common.urn.ChartUrn;
+import com.linkedin.common.urn.DashboardUrn;
 import com.linkedin.common.urn.DatasetUrn;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
@@ -19,6 +20,7 @@
     extends AbstractMultiFieldPatchBuilder<DashboardInfoPatchBuilder> {
   private static final String CHART_EDGES_PATH_START = "/chartEdges/";
   private static final String DATASET_EDGES_PATH_START = "/datasetEdges/";
+  private static final String DASHBOARDS_PATH_START = "/dashboards/";
 
   // Simplified with just Urn
   public DashboardInfoPatchBuilder addChartEdge(@Nonnull ChartUrn urn) {
@@ -36,6 +38,7 @@
     return this;
   }
 
+  // Simplified with just Urn
   public DashboardInfoPatchBuilder addDatasetEdge(@Nonnull DatasetUrn urn) {
     ObjectNode value = createEdgeValue(urn);
 
@@ -52,6 +55,22 @@
     return this;
   }
 
+  // Simplified with just Urn
+  public DashboardInfoPatchBuilder addDashboard(@Nonnull DashboardUrn urn) {
+    ObjectNode value = createEdgeValue(urn);
+
+    pathValues.add(
+        ImmutableTriple.of(PatchOperationType.ADD.getValue(), DASHBOARDS_PATH_START + urn, value));
+    return this;
+  }
+
+  public DashboardInfoPatchBuilder removeDashboard(@Nonnull DashboardUrn urn) {
+    pathValues.add(
+        ImmutableTriple.of(
+            PatchOperationType.REMOVE.getValue(), DASHBOARDS_PATH_START + urn, null));
+    return this;
+  }
+
   // Full Edge modification
   public DashboardInfoPatchBuilder addEdge(@Nonnull Edge edge) {
     ObjectNode value = createEdgeValue(edge);
@@ -87,6 +106,10 @@
       return CHART_EDGES_PATH_START + destinationUrn;
     }
 
+    if (DASHBOARD_ENTITY_NAME.equals(destinationUrn.getEntityType())) {
+      return DASHBOARDS_PATH_START + destinationUrn;
+    }
+
     // TODO: Output Data Jobs not supported by aspect, add here if this changes
 
     throw new IllegalArgumentException(

entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java

@@ -22,6 +22,7 @@ public class DashboardInfoTemplate implements ArrayMergingTemplate<DashboardInfo
   private static final String DATASETS_FIELD_NAME = "datasets";
   private static final String CHARTS_FIELD_NAME = "charts";
   private static final String DESTINATION_URN_FIELD_NAME = "destinationUrn";
+  private static final String DASHBOARDS_FIELD_NAME = "dashboards";
 
   @Override
   public DashboardInfo getSubtype(RecordTemplate recordTemplate) throws ClassCastException {
@@ -74,6 +75,12 @@ public JsonNode transformFields(JsonNode baseNode) {
             DATASET_EDGES_FIELD_NAME,
             Collections.singletonList(DESTINATION_URN_FIELD_NAME));
 
+    transformedNode =
+        arrayFieldToMap(
+            transformedNode,
+            DASHBOARDS_FIELD_NAME,
+            Collections.singletonList(DESTINATION_URN_FIELD_NAME));
+
     transformedNode =
         arrayFieldToMap(transformedNode, DATASETS_FIELD_NAME, Collections.emptyList());
 
@@ -97,6 +104,12 @@ public JsonNode rebaseFields(JsonNode patched) {
             CHART_EDGES_FIELD_NAME,
             Collections.singletonList(DESTINATION_URN_FIELD_NAME));
 
+    rebasedNode =
+        transformedMapToArray(
+            rebasedNode,
+            DASHBOARDS_FIELD_NAME,
+            Collections.singletonList(DESTINATION_URN_FIELD_NAME));
+
     rebasedNode = transformedMapToArray(rebasedNode, DATASETS_FIELD_NAME, Collections.emptyList());
     rebasedNode = transformedMapToArray(rebasedNode, CHARTS_FIELD_NAME, Collections.emptyList());
 

entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DashboardInfoTemplateTest.java

@@ -31,4 +31,26 @@ public void testDashboardInfoTemplate() throws Exception {
         UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"),
         result.getDatasetEdges().get(0).getDestinationUrn());
   }
+
+  @Test
+  public void testDashboardInfoTemplateDashboardsField() throws Exception {
+    DashboardInfoTemplate dashboardInfoTemplate = new DashboardInfoTemplate();
+    DashboardInfo dashboardInfo = dashboardInfoTemplate.getDefault();
+    JsonPatchBuilder jsonPatchBuilder = Json.createPatchBuilder();
+    jsonPatchBuilder.add(
+        "/dashboards/urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)",
+        Json.createObjectBuilder()
+            .add(
+                "destinationUrn",
+                Json.createValue(
+                    "urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)"))
+            .build());
+
+    DashboardInfo result =
+        dashboardInfoTemplate.applyPatch(dashboardInfo, jsonPatchBuilder.build());
+
+    Assert.assertEquals(
+        UrnUtils.getUrn("urn:li:dashboard:(urn:li:dataPlatform:tableau,SampleDashboard,PROD)"),
+        result.getDashboards().get(0).getDestinationUrn());
+  }
 }

metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py

@@ -102,6 +102,10 @@ def convert_dashboard_info_to_patch(
     if aspect.datasets:
         patch_builder.add_datasets(aspect.datasets)
 
+    if aspect.dashboards:
+        for dashboard in aspect.dashboards:
+            patch_builder.add_dashboard(dashboard)
+
     if aspect.access:
         patch_builder.set_access(aspect.access)
 

metadata-ingestion/src/datahub/specific/dashboard.py

@@ -161,7 +161,7 @@
                 lastModified=self._mint_auditstamp(),
             )
 
-        self._ensure_urn_type("dataset", [chart_edge], "add_chart_edge")
+        self._ensure_urn_type("chart", [chart_edge], "add_chart_edge")
         self._add_patch(
             DashboardInfo.ASPECT_NAME,
             "add",
@@ -271,6 +271,48 @@
 
         return self
 
+    def add_dashboard(
+        self, dashboard: Union[Edge, Urn, str]
+    ) -> "DashboardPatchBuilder":
+        """
+        Adds an dashboard to the DashboardPatchBuilder.
+
+        Args:
+            dashboard: The dashboard, which can be an Edge object, Urn object, or a string.
+
+        Returns:
+            The DashboardPatchBuilder instance.
+
+        Raises:
+            ValueError: If the dashboard is not a Dashboard urn.
+
+        Notes:
+            If `dashboard` is an Edge object, it is used directly. If `dashboard` is a Urn object or string,
+            it is converted to an Edge object and added with default audit stamps.
+        """
+        if isinstance(dashboard, Edge):
+            dashboard_urn: str = dashboard.destinationUrn
+            dashboard_edge: Edge = dashboard
+        elif isinstance(dashboard, (Urn, str)):
+            dashboard_urn = str(dashboard)
+            if not dashboard_urn.startswith("urn:li:dashboard:"):
+                raise ValueError(f"Input {dashboard} is not a Dashboard urn")
+
+            dashboard_edge = Edge(
+                destinationUrn=dashboard_urn,
+                created=self._mint_auditstamp(),
+                lastModified=self._mint_auditstamp(),
+            )
+
+        self._ensure_urn_type("dashboard", [dashboard_edge], "add_dashboard")
+        self._add_patch(
+            DashboardInfo.ASPECT_NAME,
+            "add",
+            path=("dashboards", dashboard_urn),
+            value=dashboard_edge,
+        )
+        return self
+
     def set_dashboard_url(
         self, dashboard_url: Optional[str]
     ) -> "DashboardPatchBuilder":

metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py

@@ -1,11 +1,19 @@
-from typing import List
+import re
+import unittest
+from typing import List, Optional
+
+import pytest
 
 import datahub.emitter.mce_builder as builder
 import datahub.metadata.schema_classes as models
 from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.incremental_lineage_helper import auto_incremental_lineage
+from datahub.ingestion.api.incremental_lineage_helper import (
+    auto_incremental_lineage,
+    convert_dashboard_info_to_patch,
+)
 from datahub.ingestion.api.source_helpers import auto_workunit
+from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.sink.file import write_metadata_file
 from tests.test_helpers import mce_helpers
 
@@ -183,3 +191,129 @@ def test_incremental_lineage_pass_through(tmp_path, pytestconfig):
     mce_helpers.check_golden_file(
         pytestconfig=pytestconfig, output_path=test_file, golden_path=golden_file
     )
+
+
+class TestConvertDashboardInfoToPatch(unittest.TestCase):
+    def setUp(self):
+        self.urn = builder.make_dashboard_urn(
+            platform="platform", name="test", platform_instance="instance"
+        )
+        self.system_metadata = models.SystemMetadataClass()
+        self.aspect = models.DashboardInfoClass(
+            title="Test Dashboard",
+            description="This is a test dashboard",
+            lastModified=models.ChangeAuditStampsClass(),
+        )
+
+    def _validate_dashboard_info_patch(
+        self, result: Optional[MetadataWorkUnit], expected_aspect_value: bytes
+    ) -> None:
+        assert (
+            result
+            and result.metadata
+            and isinstance(result.metadata, models.MetadataChangeProposalClass)
+        )
+        mcp: models.MetadataChangeProposalClass = result.metadata
+        assert (
+            mcp.entityUrn == self.urn
+            and mcp.entityType == "dashboard"
+            and mcp.aspectName == "dashboardInfo"
+            and mcp.changeType == "PATCH"
+        )
+        assert mcp.aspect and isinstance(mcp.aspect, models.GenericAspectClass)
+        aspect: models.GenericAspectClass = mcp.aspect
+        assert aspect.value == expected_aspect_value
+
+    def test_convert_dashboard_info_to_patch_with_no_additional_values(self):
+        result = convert_dashboard_info_to_patch(
+            self.urn, self.aspect, self.system_metadata
+        )
+        self._validate_dashboard_info_patch(
+            result=result,
+            expected_aspect_value=b'[{"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]',
+        )
+
+    def test_convert_dashboard_info_to_patch_with_custom_properties(self):
+        self.aspect.customProperties = {
+            "key1": "value1",
+            "key2": "value2",
+        }
+        result = convert_dashboard_info_to_patch(
+            self.urn, self.aspect, self.system_metadata
+        )
+        self._validate_dashboard_info_patch(
+            result=result,
+            expected_aspect_value=b'[{"op": "add", "path": "/customProperties/key1", "value": "value1"}, {"op": "add", "path": "/customProperties/key2", "value": "value2"}, {"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]',
+        )
+
+    def test_convert_dashboard_info_to_patch_with_chart_edges(self):
+        self.aspect.chartEdges = [
+            models.EdgeClass(
+                destinationUrn=builder.make_chart_urn(
+                    platform="platform",
+                    name="target-test",
+                    platform_instance="instance",
+                ),
+            )
+        ]
+        result = convert_dashboard_info_to_patch(
+            self.urn, self.aspect, self.system_metadata
+        )
+        self._validate_dashboard_info_patch(
+            result,
+            b'[{"op": "add", "path": "/chartEdges/urn:li:chart:(platform,instance.target-test)", "value": {"destinationUrn": "urn:li:chart:(platform,instance.target-test)"}}, {"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]',
+        )
+
+    def test_convert_dashboard_info_to_patch_with_chart_edges_no_chart(self):
+        self.aspect.chartEdges = [
+            models.EdgeClass(
+                destinationUrn=builder.make_dashboard_urn(
+                    platform="platform",
+                    name="target-test",
+                    platform_instance="instance",
+                ),
+            )
+        ]
+        with pytest.raises(
+            ValueError,
+            match=re.escape(
+                "add_chart_edge: urn:li:dashboard:(platform,instance.target-test) is not of type chart"
+            ),
+        ):
+            convert_dashboard_info_to_patch(self.urn, self.aspect, self.system_metadata)
+
+    def test_convert_dashboard_info_to_patch_with_dashboards(self):
+        self.aspect.dashboards = [
+            models.EdgeClass(
+                destinationUrn=builder.make_dashboard_urn(
+                    platform="platform",
+                    name="target-test",
+                    platform_instance="instance",
+                ),
+            )
+        ]
+        result = convert_dashboard_info_to_patch(
+            self.urn, self.aspect, self.system_metadata
+        )
+        self._validate_dashboard_info_patch(
+            result,
+            b'[{"op": "add", "path": "/title", "value": "Test Dashboard"}, {"op": "add", "path": "/description", "value": "This is a test dashboard"}, {"op": "add", "path": "/dashboards/urn:li:dashboard:(platform,instance.target-test)", "value": {"destinationUrn": "urn:li:dashboard:(platform,instance.target-test)"}}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}]',
+        )
+
+    def test_convert_dashboard_info_to_patch_with_dashboards_no_dashboard(self):
+        self.aspect.dashboards = [
+            models.EdgeClass(
+                destinationUrn=builder.make_chart_urn(
+                    platform="platform",
+                    name="target-test",
+                    platform_instance="instance",
+                ),
+            )
+        ]
+        with pytest.raises(
+            ValueError,
+            match=re.escape(
+                "add_dashboard: urn:li:chart:(platform,instance.target-test) is not of type dashboard"
+            ),
+        ):
+            convert_dashboard_info_to_patch(self.urn, self.aspect, self.system_metadata)

metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl

@@ -111,11 +111,11 @@ record DashboardInfo includes CustomProperties, ExternalReference {
       "name": "DashboardContainsDashboard",
       "entityTypes": [ "dashboard" ],
       "isLineage": true,
-      "createdOn": "datasetEdges/*/created/time"
-      "createdActor": "datasetEdges/*/created/actor"
-      "updatedOn": "datasetEdges/*/lastModified/time"
-      "updatedActor": "datasetEdges/*/lastModified/actor"
-      "properties": "datasetEdges/*/properties"
+      "createdOn": "dashboards/*/created/time"
+      "createdActor": "dashboards/*/created/actor"
+      "updatedOn": "dashboards/*/lastModified/time"
+      "updatedActor": "dashboards/*/lastModified/actor"
+      "properties": "dashboards/*/properties"
     }
   }
   dashboards: array[Edge] = [ ]