Fix#10584: Add Data Model as an entity

bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql

@@ -122,4 +122,15 @@ ALTER TABLE user_tokens MODIFY COLUMN expiryDate BIGINT UNSIGNED GENERATED ALWAY
 DELETE FROM alert_entity;
 drop table alert_action_def;
 
-ALTER TABLE alert_entity RENAME TO event_subscription_entity;
+ALTER TABLE alert_entity RENAME TO event_subscription_entity;
+-- create data model table
+CREATE TABLE IF NOT EXISTS data_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);

bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql

@@ -126,3 +126,15 @@ DELETE FROM alert_entity;
 drop table alert_action_def;
 
 ALTER TABLE alert_entity RENAME TO event_subscription_entity;
+
+-- create data model table
+CREATE TABLE IF NOT EXISTS data_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);

ingestion/examples/sample_data/dashboards/charts.json

@@ -4,7 +4,7 @@
 			"id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e",
 			"name": "114",
 			"displayName": "# of Games That Hit 100k in Sales By Release Year",
-			"fullyQualifiedName": "local_superset.101",
+			"fullyQualifiedName": "sample_superset.101",
 			"description": "",
 			"chartId": "114",
 			"chartType": "Area",
@@ -15,7 +15,7 @@
 			"id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4",
 			"name":"166",
 			"displayName": "% Rural",
-			"fullyQualifiedName": "local_superset.110",
+			"fullyQualifiedName": "sample_superset.110",
 			"description": "",
 			"chartId": "166",
 			"chartType": "Other",
@@ -26,7 +26,7 @@
 			"id": "22b95748-4a7b-48ad-859e-cf7c66a7f343",
 			"name": "92",
 			"displayName": "✈️ Relocation ability",
-			"fullyQualifiedName": "local_superset.92",
+			"fullyQualifiedName": "sample_superset.92",
 			"description": "",
 			"chartId": "92",
 			"chartType": "Other",
@@ -37,7 +37,7 @@
 			"id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da",
 			"name": "117",
 			"displayName": "Age distribution of respondents",
-			"fullyQualifiedName": "local_superset.11",
+			"fullyQualifiedName": "sample_superset.11",
 			"description": "",
 			"chartId": "117",
 			"chartType": "Histogram",
@@ -47,7 +47,7 @@
 		{
 			"id": "57944482-e187-439a-aaae-0e8aabd2f455",
 			"displayName": "Arcs",
-			"fullyQualifiedName": "local_superset.197",
+			"fullyQualifiedName": "sample_superset.197",
 			"description": "",
 			"name": "197",
 			"chartType": "Other",
@@ -57,7 +57,7 @@
 		{
 			"id": "d88e2056-c74a-410d-829e-eb31b040c132",
 			"displayName": "Are you an ethnic minority in your city?",
-			"fullyQualifiedName": "local_superset.127",
+			"fullyQualifiedName": "sample_superset.127",
 			"description": "",
 			"name": "127",
 			"chartType": "Other",
@@ -67,7 +67,7 @@
 		{
 			"id": "c1d3e156-4628-414e-8d6e-a6bdd486128f",
 			"displayName": "Average and Sum Trends",
-			"fullyQualifiedName": "local_superset.183",
+			"fullyQualifiedName": "sample_superset.183",
 			"description": "",
 			"name": "183",
 			"chartType": "Line",
@@ -77,7 +77,7 @@
 		{
 			"id": "bfc57519-8cef-47e6-a423-375d5b89a6a4",
 			"displayName": "Birth in France by department in 2016",
-			"fullyQualifiedName": "local_superset.Birth in France by department in 2016",
+			"fullyQualifiedName": "sample_superset.Birth in France by department in 2016",
 			"description": "",
 			"name": "161",
 			"chartType": "Other",
@@ -87,7 +87,7 @@
 		{
 			"id": "bf2eeac4-7226-46c6-bbef-918569c137a0",
 			"displayName": "Box plot",
-			"fullyQualifiedName": "local_superset.170",
+			"fullyQualifiedName": "sample_superset.170",
 			"description": "",
 			"name": "170",
 			"chartType": "Bar",
@@ -97,7 +97,7 @@
 		{
 			"id": "167fd63b-42f1-4d7e-a37d-893fd8173b44",
 			"displayName": "Boy Name Cloud",
-			"fullyQualifiedName": "local_superset.180",
+			"fullyQualifiedName": "sample_superset.180",
 			"description": "",
 			"name": "180",
 			"chartType": "Other",
@@ -107,7 +107,7 @@
 		{
 			"id": "8474e579-4eff-492b-8685-70ec9aa99f5f",
 			"displayName": "ETA Predictions Accuracy",
-			"fullyQualifiedName": "local_superset.210",
+			"fullyQualifiedName": "sample_superset.210",
 			"description": "",
 			"name": "210",
 			"chartType": "Line",
@@ -117,7 +117,7 @@
 		{
 			"id": "12345e567-4eff-492b-8685-69ec9bb88g4g",
 			"displayName": "Sales Predictions Accuracy",
-			"fullyQualifiedName": "local_superset.211",
+			"fullyQualifiedName": "sample_superset.211",
 			"description": "",
 			"name": "211",
 			"chartType": "Line",

ingestion/examples/sample_data/dashboards/dashboards.json

@@ -4,7 +4,7 @@
       "id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f",
       "name": "8",
       "displayName": "Orders dashboard",
-      "fullyQualifiedName": "local_superset.8",
+      "fullyQualifiedName": "sample_superset.8",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/1/",
       "charts": ["sample_superset.183", "sample_superset.170", "sample_superset.197"],
@@ -14,17 +14,18 @@
       "id": "063cd787-8630-4809-9702-34d3992c7248",
       "name": "9",
       "displayName": "COVID Vaccine Dashboard",
-      "fullyQualifiedName": "local_superset.9",
+      "fullyQualifiedName": "sample_superset.9",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/8/",
       "charts": ["sample_superset.117", "sample_superset.197"],
+      "dataModels": ["sample_superset.model.103", "sample_superset.model.73"],
       "href": "http://localhost:8585/api/v1/dashboards/063cd787-8630-4809-9702-34d3992c7248"
     },
     {
       "id": "df6c698e-066a-4440-be0a-121025573b73",
       "name": "10",
       "displayName": "deck.gl Demo",
-      "fullyQualifiedName": "local_superset.10",
+      "fullyQualifiedName": "sample_superset.10",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/deck/",
       "charts": ["sample_superset.127", "sample_superset.166", "sample_superset.114"],
@@ -34,7 +35,7 @@
       "id": "98b38a49-b5c6-431b-b61f-690e39f8ead2",
       "name": "11",
       "displayName": "FCC New Coder Survey 2018",
-      "fullyQualifiedName": "local_superset.11",
+      "fullyQualifiedName": "sample_superset.11",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/7/",
       "charts": ["sample_superset.183", "sample_superset.197", "sample_superset.170", "sample_superset.180"],
@@ -44,7 +45,7 @@
       "id": "dffcf9b2-4f43-4881-a5f5-10109655bf50",
       "name": "12",
       "displayName": "Misc Charts",
-      "fullyQualifiedName": "local_superset.12",
+      "fullyQualifiedName": "sample_superset.12",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/",
       "charts": ["sample_superset.127", "sample_superset.197"],
@@ -54,7 +55,7 @@
       "id": "2583737d-6236-421e-ba0f-cd0b79adb216",
       "name": "31",
       "displayName": "Sales Dashboard",
-      "fullyQualifiedName": "local_superset.31",
+      "fullyQualifiedName": "sample_superset.31",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/6/",
       "charts": ["sample_superset.92", "sample_superset.117", "sample_superset.166"],
@@ -64,7 +65,7 @@
       "id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2",
       "name": "33",
       "displayName": "Slack Dashboard",
-      "fullyQualifiedName": "local_superset.33",
+      "fullyQualifiedName": "sample_superset.33",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/10/",
       "charts": ["sample_superset.114", "sample_superset.92", "sample_superset.127"],
@@ -74,7 +75,7 @@
       "id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45",
       "name": "34",
       "displayName": "Unicode Test",
-      "fullyQualifiedName": "local_superset.34",
+      "fullyQualifiedName": "sample_superset.34",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/",
       "charts": ["sample_superset.161", "sample_superset.170", "sample_superset.180"],
@@ -84,7 +85,7 @@
       "id": "a3ace318-ee37-4da1-974a-62eddbd77d20",
       "name": "45",
       "displayName": "USA Births Names",
-      "fullyQualifiedName": "local_superset.45",
+      "fullyQualifiedName": "sample_superset.45",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/births/",
       "charts": ["sample_superset.180"],
@@ -94,7 +95,7 @@
       "id": "e6e21717-1164-403f-8807-d12be277aec6",
       "name": "51",
       "displayName": "Video Game Sales",
-      "fullyQualifiedName": "local_superset.51",
+      "fullyQualifiedName": "sample_superset.51",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/11/",
       "charts": ["sample_superset.127", "sample_superset.183"],
@@ -104,7 +105,7 @@
       "id": "d2b0af00-f419-4905-bb43-036697ce53a5",
       "name": "eta_predictions_performance",
       "displayName": "ETA Predictions Performance",
-      "fullyQualifiedName": "local_superset.eta_predictions_performance",
+      "fullyQualifiedName": "sample_superset.eta_predictions_performance",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/eta_predictions_performance/",
       "charts": ["sample_superset.210"],
@@ -114,7 +115,7 @@
       "id": "f5a1af99-f123-7845-cc12-0312347ce53a",
       "name": "forecast_sales_performance",
       "displayName": "ETA Predictions Performance",
-      "fullyQualifiedName": "local_superset.forecast_sales_performance",
+      "fullyQualifiedName": "sample_superset.forecast_sales_performance",
       "description": "",
       "dashboardUrl": "http://localhost:808/superset/dashboard/forecast_sales_performance/",
       "charts": ["sample_superset.211"],

ingestion/examples/sample_data/dashboards/dataModels.json

@@ -0,0 +1,101 @@
+{
+  "datamodels": [
+    {
+      "id": "e093dd27-390e-4360-8efd-e4d63ec167a9",
+      "name": "103",
+      "displayName": "Vaccine Candidates per Phase",
+      "fullyQualifiedName": "sample_superset.model.103",
+      "description": "Data of Vaccine Candidates per Phase",
+      "version": 0.1,
+      "updatedAt": 1638354087591,
+      "dataModelType":  "SupersetDataModel",
+      "serviceType": "Superset",
+      "sql":  "SELECT CASE\n           WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n           WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n           WHEN stage_of_development = 'Phase I/II'\n                or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n           WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n           WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n       END AS clinical_stage,\n       COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n             WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n             WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n             WHEN stage_of_development = 'Phase I/II'\n                  or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n             WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n             WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n         END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n",
+      "columns": [
+        {
+          "name": "0. Pre-clinical",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Vaccine Candidates in phase: 'Pre-clinical'",
+          "tags": [],
+          "ordinalPosition": 1
+        },
+        {
+          "name": "2. Phase II or Combined I/II",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'",
+          "tags": [],
+          "ordinalPosition": 2
+        },
+        {
+          "name": "1. Phase I",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Vaccine Candidates in phase: 'Phase I'",
+          "tags": [],
+          "ordinalPosition": 3
+        },
+        {
+          "name": "3. Phase III",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Vaccine Candidates in phase: 'Phase III'",
+          "tags": [],
+          "ordinalPosition": 4
+        },
+        {
+          "name": "4. Authorized",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Vaccine Candidates in phase: 'Authorize'",
+          "tags": [],
+          "ordinalPosition": 5
+        }
+      ],
+      "tags": [],
+      "followers": []
+    },
+    {
+      "id": "5bab1ca3-7a22-4f21-9b34-e2c44dee1af6",
+      "name": "73",
+      "displayName": "Vaccine Candidates per Country",
+      "fullyQualifiedName": "sample_superset.model.73",
+      "description": "Data of Vaccine Candidates per Country",
+      "version": 0.1,
+      "updatedAt": 1638354087591,
+      "dataModelType":  "SupersetDataModel",
+      "serviceType": "Superset",
+      "sql":  "SELECT country_name AS country_name,\n       COUNT(*) AS count,\n       count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;",
+      "columns": [
+        {
+          "name": "country_name",
+          "dataType": "VARCHAR",
+          "dataTypeDisplay": "varchar",
+          "dataLength": 256,
+          "description": "Name of the country.",
+          "tags": [],
+          "ordinalPosition": 1
+        },
+        {
+          "name": "count",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Total number of vaccine candidates per country.",
+          "tags": [],
+          "ordinalPosition": 2
+        },
+        {
+          "name": "COUNT(Country_Name)",
+          "dataType": "NUMERIC",
+          "dataTypeDisplay": "numeric",
+          "description": "Total number of vaccine candidates.",
+          "tags": [],
+          "ordinalPosition": 3
+        }
+      ],
+      "tags": [],
+      "followers": []
+    }
+  ]
+}

ingestion/examples/sample_data/lineage/lineage.json

@@ -58,6 +58,27 @@
     },
     "edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
     "sql_query": "create ecommerce_db.shopify.\"dim.product.variant\" as select * from sample_data.ecommerce_db.shopify.raw_customer"
-
+  },
+  {
+    "from": {
+      "fqn": "sample_superset.model.103",
+      "type": "dataModel"
+    },
+    "to": {
+      "fqn": "sample_superset.9",
+      "type": "dashboard"
+    },
+    "edge_meta": { "fqn": "", "type": "" }
+  },
+  {
+    "from": {
+      "fqn": "sample_superset.model.73",
+      "type": "dataModel"
+    },
+    "to": {
+      "fqn": "sample_superset.9",
+      "type": "dashboard"
+    },
+    "edge_meta": { "fqn": "", "type": "" }
   }
 ]