feat: support dashboard chart in search (#383)

* feat: support dashboard chart in search Signed-off-by: feng-tao <[email protected]> * feat: fix lint Signed-off-by: feng-tao <[email protected]> * update Signed-off-by: feng-tao <[email protected]>
amundsen-io · Oct 15, 2020 · 6cced36 · 6cced36
1 parent 8beee3e
commit 6cced36
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 26 deletions.
diff --git a/databuilder/extractor/neo4j_search_data_extractor.py b/databuilder/extractor/neo4j_search_data_extractor.py
@@ -84,32 +84,33 @@ class Neo4jSearchDataExtractor(Extractor):
 
     DEFAULT_NEO4J_DASHBOARD_CYPHER_QUERY = textwrap.dedent(
         """
-        MATCH (dashboard:Dashboard)
-        {publish_tag_filter}
-        MATCH (dashboard)-[:DASHBOARD_OF]->(dbg:Dashboardgroup)
-        MATCH (dbg)-[:DASHBOARD_GROUP_OF]->(cluster:Cluster)
-        OPTIONAL MATCH (dashboard)-[:DESCRIPTION]->(db_descr:Description)
-        OPTIONAL MATCH (dbg)-[:DESCRIPTION]->(dbg_descr:Description)
-        OPTIONAL MATCH (dashboard)-[:EXECUTED]->(last_exec:Execution)
-        WHERE split(last_exec.key, '/')[5] = '_last_successful_execution'
-        OPTIONAL MATCH (dashboard)-[read:READ_BY]->(user:User)
-        WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, SUM(read.read_count) AS total_usage
-        OPTIONAL MATCH (dashboard)-[:HAS_QUERY]->(query:Query)
-        WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, COLLECT(DISTINCT query.name) as query_names,
-        total_usage
-        OPTIONAL MATCH (dashboard)-[:TAGGED_BY]->(tags:Tag) WHERE tags.tag_type='default'
-        WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, query_names, total_usage,
-        COLLECT(DISTINCT tags.key) as tags
-        OPTIONAL MATCH (dashboard)-[:HAS_BADGE]->(badges:Badge)
-        WITH  dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, query_names, total_usage, tags,
-        COLLECT(DISTINCT badges.key) as badges
-        RETURN dbg.name as group_name, dashboard.name as name, cluster.name as cluster,
-        coalesce(db_descr.description, '') as description,
-        coalesce(dbg.description, '') as group_description, dbg.dashboard_group_url as group_url,
-        dashboard.dashboard_url as url, dashboard.key as uri,
-        split(dashboard.key, '_')[0] as product, toInteger(last_exec.timestamp) as last_successful_run_timestamp,
-        query_names, total_usage, tags, badges
-        order by dbg.name;
+         MATCH (dashboard:Dashboard)
+         {publish_tag_filter}
+         MATCH (dashboard)-[:DASHBOARD_OF]->(dbg:Dashboardgroup)
+         MATCH (dbg)-[:DASHBOARD_GROUP_OF]->(cluster:Cluster)
+         OPTIONAL MATCH (dashboard)-[:DESCRIPTION]->(db_descr:Description)
+         OPTIONAL MATCH (dbg)-[:DESCRIPTION]->(dbg_descr:Description)
+         OPTIONAL MATCH (dashboard)-[:EXECUTED]->(last_exec:Execution)
+         WHERE split(last_exec.key, '/')[5] = '_last_successful_execution'
+         OPTIONAL MATCH (dashboard)-[read:READ_BY]->(user:User)
+         WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, SUM(read.read_count) AS total_usage
+         OPTIONAL MATCH (dashboard)-[:HAS_QUERY]->(query:Query)-[:HAS_CHART]->(chart:Chart)
+         WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, COLLECT(DISTINCT query.name) as query_names,
+         COLLECT(DISTINCT chart.name) as chart_names,
+         total_usage
+         OPTIONAL MATCH (dashboard)-[:TAGGED_BY]->(tags:Tag) WHERE tags.tag_type='default'
+         WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, query_names, chart_names, total_usage,
+         COLLECT(DISTINCT tags.key) as tags
+         OPTIONAL MATCH (dashboard)-[:HAS_BADGE]->(badges:Badge)
+         WITH  dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, query_names, chart_names, total_usage, tags,
+         COLLECT(DISTINCT badges.key) as badges
+         RETURN dbg.name as group_name, dashboard.name as name, cluster.name as cluster,
+         coalesce(db_descr.description, '') as description,
+         coalesce(dbg.description, '') as group_description, dbg.dashboard_group_url as group_url,
+         dashboard.dashboard_url as url, dashboard.key as uri,
+         split(dashboard.key, '_')[0] as product, toInteger(last_exec.timestamp) as last_successful_run_timestamp,
+         query_names, chart_names, total_usage, tags, badges
+         order by dbg.name
         """
     )
 

diff --git a/databuilder/models/dashboard_elasticsearch_document.py b/databuilder/models/dashboard_elasticsearch_document.py
@@ -20,6 +20,7 @@ def __init__(self,
                  cluster: Optional[str] = '',
                  group_description: Optional[str] = None,
                  query_names: Union[List[str], None] = None,
+                 chart_names: Optional[List[str]] = None,
                  group_url: Optional[str] = None,
                  url: Optional[str] = None,
                  uri: Optional[str] = None,
@@ -39,5 +40,6 @@ def __init__(self,
         self.total_usage = total_usage
         self.group_description = group_description
         self.query_names = query_names
+        self.chart_names = chart_names
         self.tags = tags
         self.badges = badges
diff --git a/databuilder/publisher/elasticsearch_constants.py b/databuilder/publisher/elasticsearch_constants.py
@@ -166,6 +166,15 @@
                     }
                   }
                 },
+                "chart_names": {
+                  "type":"text",
+                  "analyzer": "simple",
+                  "fields": {
+                    "raw": {
+                      "type": "keyword"
+                    }
+                  }
+                },
                 "tags": {
                   "type": "keyword"
                 },

diff --git a/tests/unit/models/test_dashboard_elasticsearch_document.py b/tests/unit/models/test_dashboard_elasticsearch_document.py
@@ -20,6 +20,7 @@ def test_to_json(self) -> None:
                                        cluster='gold',
                                        group_description='work space group',
                                        query_names=['query1'],
+                                       chart_names=['chart1'],
                                        group_url='mode_group_url',
                                        url='mode_report_url',
                                        uri='mode_dashboard://gold.cluster/dashboard_group/dashboard',
@@ -37,6 +38,7 @@ def test_to_json(self) -> None:
                                   "url": "mode_report_url",
                                   "uri": "mode_dashboard://gold.cluster/dashboard_group/dashboard",
                                   "query_names": ['query1'],
+                                  "chart_names": ['chart1'],
                                   "last_successful_run_timestamp": 10,
                                   "group_description": "work space group",
                                   "total_usage": 10,