Skip to content

Commit

Permalink
Fix#10584: Add Data Model as an entity (#10636)
Browse files Browse the repository at this point in the history
* Add Data Model as entity

* Add sample_data + update dashboard resource and repository with data models

* Fix Java style

* Addess PR comments

* Update bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql

* Pylint error

* Address PR comments

* Address PR comments

* Address PR comments

* Minor change

* Fix error in sample_data

* Fix failing test

* Add missing resource and event sub descriptors
  • Loading branch information
nahuelverdugo authored Mar 28, 2023
1 parent 86febae commit bea38d7
Show file tree
Hide file tree
Showing 30 changed files with 1,381 additions and 137 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,15 @@ ALTER TABLE user_tokens MODIFY COLUMN expiryDate BIGINT UNSIGNED GENERATED ALWAY
DELETE FROM alert_entity;
drop table alert_action_def;

ALTER TABLE alert_entity RENAME TO event_subscription_entity;
ALTER TABLE alert_entity RENAME TO event_subscription_entity;
-- create data model table
CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,15 @@ DELETE FROM alert_entity;
drop table alert_action_def;

ALTER TABLE alert_entity RENAME TO event_subscription_entity;

-- create data model table
CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);
24 changes: 12 additions & 12 deletions ingestion/examples/sample_data/dashboards/charts.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e",
"name": "114",
"displayName": "# of Games That Hit 100k in Sales By Release Year",
"fullyQualifiedName": "local_superset.101",
"fullyQualifiedName": "sample_superset.101",
"description": "",
"chartId": "114",
"chartType": "Area",
Expand All @@ -15,7 +15,7 @@
"id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4",
"name":"166",
"displayName": "% Rural",
"fullyQualifiedName": "local_superset.110",
"fullyQualifiedName": "sample_superset.110",
"description": "",
"chartId": "166",
"chartType": "Other",
Expand All @@ -26,7 +26,7 @@
"id": "22b95748-4a7b-48ad-859e-cf7c66a7f343",
"name": "92",
"displayName": "✈️ Relocation ability",
"fullyQualifiedName": "local_superset.92",
"fullyQualifiedName": "sample_superset.92",
"description": "",
"chartId": "92",
"chartType": "Other",
Expand All @@ -37,7 +37,7 @@
"id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da",
"name": "117",
"displayName": "Age distribution of respondents",
"fullyQualifiedName": "local_superset.11",
"fullyQualifiedName": "sample_superset.11",
"description": "",
"chartId": "117",
"chartType": "Histogram",
Expand All @@ -47,7 +47,7 @@
{
"id": "57944482-e187-439a-aaae-0e8aabd2f455",
"displayName": "Arcs",
"fullyQualifiedName": "local_superset.197",
"fullyQualifiedName": "sample_superset.197",
"description": "",
"name": "197",
"chartType": "Other",
Expand All @@ -57,7 +57,7 @@
{
"id": "d88e2056-c74a-410d-829e-eb31b040c132",
"displayName": "Are you an ethnic minority in your city?",
"fullyQualifiedName": "local_superset.127",
"fullyQualifiedName": "sample_superset.127",
"description": "",
"name": "127",
"chartType": "Other",
Expand All @@ -67,7 +67,7 @@
{
"id": "c1d3e156-4628-414e-8d6e-a6bdd486128f",
"displayName": "Average and Sum Trends",
"fullyQualifiedName": "local_superset.183",
"fullyQualifiedName": "sample_superset.183",
"description": "",
"name": "183",
"chartType": "Line",
Expand All @@ -77,7 +77,7 @@
{
"id": "bfc57519-8cef-47e6-a423-375d5b89a6a4",
"displayName": "Birth in France by department in 2016",
"fullyQualifiedName": "local_superset.Birth in France by department in 2016",
"fullyQualifiedName": "sample_superset.Birth in France by department in 2016",
"description": "",
"name": "161",
"chartType": "Other",
Expand All @@ -87,7 +87,7 @@
{
"id": "bf2eeac4-7226-46c6-bbef-918569c137a0",
"displayName": "Box plot",
"fullyQualifiedName": "local_superset.170",
"fullyQualifiedName": "sample_superset.170",
"description": "",
"name": "170",
"chartType": "Bar",
Expand All @@ -97,7 +97,7 @@
{
"id": "167fd63b-42f1-4d7e-a37d-893fd8173b44",
"displayName": "Boy Name Cloud",
"fullyQualifiedName": "local_superset.180",
"fullyQualifiedName": "sample_superset.180",
"description": "",
"name": "180",
"chartType": "Other",
Expand All @@ -107,7 +107,7 @@
{
"id": "8474e579-4eff-492b-8685-70ec9aa99f5f",
"displayName": "ETA Predictions Accuracy",
"fullyQualifiedName": "local_superset.210",
"fullyQualifiedName": "sample_superset.210",
"description": "",
"name": "210",
"chartType": "Line",
Expand All @@ -117,7 +117,7 @@
{
"id": "12345e567-4eff-492b-8685-69ec9bb88g4g",
"displayName": "Sales Predictions Accuracy",
"fullyQualifiedName": "local_superset.211",
"fullyQualifiedName": "sample_superset.211",
"description": "",
"name": "211",
"chartType": "Line",
Expand Down
101 changes: 101 additions & 0 deletions ingestion/examples/sample_data/dashboards/dashboardDataModels.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
{
"datamodels": [
{
"id": "e093dd27-390e-4360-8efd-e4d63ec167a9",
"name": "103",
"displayName": "Vaccine Candidates per Phase",
"fullyQualifiedName": "sample_superset.model.103",
"description": "Data of Vaccine Candidates per Phase",
"version": 0.1,
"updatedAt": 1638354087591,
"dataModelType": "SupersetDataModel",
"serviceType": "Superset",
"sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n",
"columns": [
{
"name": "0. Pre-clinical",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Pre-clinical'",
"tags": [],
"ordinalPosition": 1
},
{
"name": "2. Phase II or Combined I/II",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'",
"tags": [],
"ordinalPosition": 2
},
{
"name": "1. Phase I",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Phase I'",
"tags": [],
"ordinalPosition": 3
},
{
"name": "3. Phase III",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Phase III'",
"tags": [],
"ordinalPosition": 4
},
{
"name": "4. Authorized",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Authorize'",
"tags": [],
"ordinalPosition": 5
}
],
"tags": [],
"followers": []
},
{
"id": "5bab1ca3-7a22-4f21-9b34-e2c44dee1af6",
"name": "73",
"displayName": "Vaccine Candidates per Country",
"fullyQualifiedName": "sample_superset.model.73",
"description": "Data of Vaccine Candidates per Country",
"version": 0.1,
"updatedAt": 1638354087591,
"dataModelType": "SupersetDataModel",
"serviceType": "Superset",
"sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;",
"columns": [
{
"name": "country_name",
"dataType": "VARCHAR",
"dataTypeDisplay": "varchar",
"dataLength": 256,
"description": "Name of the country.",
"tags": [],
"ordinalPosition": 1
},
{
"name": "count",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Total number of vaccine candidates per country.",
"tags": [],
"ordinalPosition": 2
},
{
"name": "COUNT(Country_Name)",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Total number of vaccine candidates.",
"tags": [],
"ordinalPosition": 3
}
],
"tags": [],
"followers": []
}
]
}
25 changes: 13 additions & 12 deletions ingestion/examples/sample_data/dashboards/dashboards.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f",
"name": "8",
"displayName": "Orders dashboard",
"fullyQualifiedName": "local_superset.8",
"fullyQualifiedName": "sample_superset.8",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/1/",
"charts": ["sample_superset.183", "sample_superset.170", "sample_superset.197"],
Expand All @@ -14,17 +14,18 @@
"id": "063cd787-8630-4809-9702-34d3992c7248",
"name": "9",
"displayName": "COVID Vaccine Dashboard",
"fullyQualifiedName": "local_superset.9",
"fullyQualifiedName": "sample_superset.9",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/8/",
"charts": ["sample_superset.117", "sample_superset.197"],
"dataModels": ["sample_superset.model.103", "sample_superset.model.73"],
"href": "http://localhost:8585/api/v1/dashboards/063cd787-8630-4809-9702-34d3992c7248"
},
{
"id": "df6c698e-066a-4440-be0a-121025573b73",
"name": "10",
"displayName": "deck.gl Demo",
"fullyQualifiedName": "local_superset.10",
"fullyQualifiedName": "sample_superset.10",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/deck/",
"charts": ["sample_superset.127", "sample_superset.166", "sample_superset.114"],
Expand All @@ -34,7 +35,7 @@
"id": "98b38a49-b5c6-431b-b61f-690e39f8ead2",
"name": "11",
"displayName": "FCC New Coder Survey 2018",
"fullyQualifiedName": "local_superset.11",
"fullyQualifiedName": "sample_superset.11",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/7/",
"charts": ["sample_superset.183", "sample_superset.197", "sample_superset.170", "sample_superset.180"],
Expand All @@ -44,7 +45,7 @@
"id": "dffcf9b2-4f43-4881-a5f5-10109655bf50",
"name": "12",
"displayName": "Misc Charts",
"fullyQualifiedName": "local_superset.12",
"fullyQualifiedName": "sample_superset.12",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/",
"charts": ["sample_superset.127", "sample_superset.197"],
Expand All @@ -54,7 +55,7 @@
"id": "2583737d-6236-421e-ba0f-cd0b79adb216",
"name": "31",
"displayName": "Sales Dashboard",
"fullyQualifiedName": "local_superset.31",
"fullyQualifiedName": "sample_superset.31",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/6/",
"charts": ["sample_superset.92", "sample_superset.117", "sample_superset.166"],
Expand All @@ -64,7 +65,7 @@
"id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2",
"name": "33",
"displayName": "Slack Dashboard",
"fullyQualifiedName": "local_superset.33",
"fullyQualifiedName": "sample_superset.33",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/10/",
"charts": ["sample_superset.114", "sample_superset.92", "sample_superset.127"],
Expand All @@ -74,7 +75,7 @@
"id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45",
"name": "34",
"displayName": "Unicode Test",
"fullyQualifiedName": "local_superset.34",
"fullyQualifiedName": "sample_superset.34",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/",
"charts": ["sample_superset.161", "sample_superset.170", "sample_superset.180"],
Expand All @@ -84,7 +85,7 @@
"id": "a3ace318-ee37-4da1-974a-62eddbd77d20",
"name": "45",
"displayName": "USA Births Names",
"fullyQualifiedName": "local_superset.45",
"fullyQualifiedName": "sample_superset.45",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/births/",
"charts": ["sample_superset.180"],
Expand All @@ -94,7 +95,7 @@
"id": "e6e21717-1164-403f-8807-d12be277aec6",
"name": "51",
"displayName": "Video Game Sales",
"fullyQualifiedName": "local_superset.51",
"fullyQualifiedName": "sample_superset.51",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/11/",
"charts": ["sample_superset.127", "sample_superset.183"],
Expand All @@ -104,7 +105,7 @@
"id": "d2b0af00-f419-4905-bb43-036697ce53a5",
"name": "eta_predictions_performance",
"displayName": "ETA Predictions Performance",
"fullyQualifiedName": "local_superset.eta_predictions_performance",
"fullyQualifiedName": "sample_superset.eta_predictions_performance",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/eta_predictions_performance/",
"charts": ["sample_superset.210"],
Expand All @@ -114,7 +115,7 @@
"id": "f5a1af99-f123-7845-cc12-0312347ce53a",
"name": "forecast_sales_performance",
"displayName": "ETA Predictions Performance",
"fullyQualifiedName": "local_superset.forecast_sales_performance",
"fullyQualifiedName": "sample_superset.forecast_sales_performance",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/forecast_sales_performance/",
"charts": ["sample_superset.211"],
Expand Down
23 changes: 22 additions & 1 deletion ingestion/examples/sample_data/lineage/lineage.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,27 @@
},
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
"sql_query": "create ecommerce_db.shopify.\"dim.product.variant\" as select * from sample_data.ecommerce_db.shopify.raw_customer"

},
{
"from": {
"fqn": "sample_superset.model.103",
"type": "dashboardDataModel"
},
"to": {
"fqn": "sample_superset.9",
"type": "dashboard"
},
"edge_meta": { "fqn": "", "type": "" }
},
{
"from": {
"fqn": "sample_superset.model.73",
"type": "dashboardDataModel"
},
"to": {
"fqn": "sample_superset.9",
"type": "dashboard"
},
"edge_meta": { "fqn": "", "type": "" }
}
]
Loading

0 comments on commit bea38d7

Please sign in to comment.