diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py index 86f518801..6f192ac09 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py @@ -10,6 +10,8 @@ from databuilder.rest_api.rest_api_query import RestApiQuery from databuilder.transformer.base_transformer import ChainedTransformer from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS +from databuilder.transformer.regex_str_replace_transformer import RegexStrReplaceTransformer, \ + REGEX_REPLACE_TUPLE_LIST, ATTRIBUTE_NAME from databuilder.transformer.template_variable_substitution_transformer import \ TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME @@ -44,6 +46,14 @@ def init(self, conf): transformers.append(variable_substitution_transformer) + # Escape backslash as it breaks Cypher statement. + replace_transformer = RegexStrReplaceTransformer() + replace_transformer.init( + conf=Scoped.get_scoped_conf(self._conf, replace_transformer.get_scope()).with_fallback( + ConfigFactory.from_dict( + {REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')], ATTRIBUTE_NAME: 'query_text'}))) + transformers.append(replace_transformer) + dict_to_model_transformer = DictToModel() dict_to_model_transformer.init( conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback( @@ -86,8 +96,8 @@ def _build_restapi_query(self): json_path=json_path, field_names=field_names, skip_no_result=True) queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries' - json_path = '_embedded.queries[*].[token,name]' - field_names = ['query_id', 'query_name'] + json_path = '_embedded.queries[*].[token,name,raw_query]' + field_names = ['query_id', 'query_name', 'query_text'] query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params, json_path=json_path, field_names=field_names, skip_no_result=True) diff --git a/databuilder/models/dashboard/dashboard_query.py b/databuilder/models/dashboard/dashboard_query.py index cdd1a6c8a..b148efcd9 100644 --- a/databuilder/models/dashboard/dashboard_query.py +++ b/databuilder/models/dashboard/dashboard_query.py @@ -26,6 +26,7 @@ def __init__(self, query_name, # type: str query_id=None, # type: Optional[str] url='', # type: Optional[str] + query_text=None, # type: Optional[str] product='', # type: Optional[str] cluster='gold', # type: str **kwargs @@ -35,6 +36,7 @@ def __init__(self, self._query_name = query_name self._query_id = query_id if query_id else query_name self._url = url + self._query_text = query_text self._product = product self._cluster = cluster self._node_iterator = self._create_node_iterator() @@ -59,6 +61,9 @@ def _create_node_iterator(self): # noqa: C901 if self._url: node['url'] = self._url + if self._query_text: + node['query_text'] = self._query_text + yield node def create_next_relation(self): @@ -94,12 +99,13 @@ def _get_query_node_key(self): ) def __repr__(self): - return 'DashboardQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format( + return 'DashboardQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format( self._dashboard_group_id, self._dashboard_id, self._query_name, self._query_id, self._url, + self._query_text, self._product, self._cluster ) diff --git a/databuilder/transformer/regex_str_replace_transformer.py b/databuilder/transformer/regex_str_replace_transformer.py index 202bf1d5b..47b8a0cb7 100644 --- a/databuilder/transformer/regex_str_replace_transformer.py +++ b/databuilder/transformer/regex_str_replace_transformer.py @@ -28,7 +28,11 @@ def init(self, conf): def transform(self, record): # type: (Any) -> Any - val = getattr(record, self._attribute_name) + + if isinstance(record, dict): + val = record.get(self._attribute_name) + else: + val = getattr(record, self._attribute_name) if val is None or not isinstance(val, six.string_types): return record @@ -40,7 +44,11 @@ def transform(self, record): for regex_replace_tuple in self._regex_replace_tuples: val = val.replace(regex_replace_tuple[0], regex_replace_tuple[1]) - setattr(record, self._attribute_name, val) + if isinstance(record, dict): + record[self._attribute_name] = val + else: + setattr(record, self._attribute_name, val) + return record def get_scope(self): diff --git a/setup.py b/setup.py index 09c5914e2..1d3e37e34 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -__version__ = '2.5.17' +__version__ = '2.5.18' requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt') with open(requirements_path) as requirements_file: diff --git a/tests/unit/models/dashboard/test_dashboard_query.py b/tests/unit/models/dashboard/test_dashboard_query.py index 9dff43f9b..798793d17 100644 --- a/tests/unit/models/dashboard/test_dashboard_query.py +++ b/tests/unit/models/dashboard/test_dashboard_query.py @@ -15,10 +15,12 @@ def test_create_nodes(self): dashboard_id='d_id', query_id='q_id', query_name='q_name', - url='http://foo.bar/query/baz') + url='http://foo.bar/query/baz', + query_text='SELECT * FROM foo.bar') actual = dashboard_query.create_next_node() expected = {'url': 'http://foo.bar/query/baz', 'name': 'q_name', 'id': 'q_id', + 'query_text': 'SELECT * FROM foo.bar', NODE_KEY: '_dashboard://gold.dg_id/d_id/query/q_id', NODE_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL} diff --git a/tests/unit/transformer/test_regex_str_replace_transformer.py b/tests/unit/transformer/test_regex_str_replace_transformer.py index cd7caa6c0..6de48da01 100644 --- a/tests/unit/transformer/test_regex_str_replace_transformer.py +++ b/tests/unit/transformer/test_regex_str_replace_transformer.py @@ -2,7 +2,8 @@ from pyhocon import ConfigFactory -from databuilder.transformer.regex_str_replace_transformer import RegexStrReplaceTransformer +from databuilder.transformer.regex_str_replace_transformer import RegexStrReplaceTransformer, \ + REGEX_REPLACE_TUPLE_LIST, ATTRIBUTE_NAME class TestRegexReplacement(unittest.TestCase): @@ -37,8 +38,8 @@ def test_none_val(self): def _default_test_transformer(self): # type: () -> RegexStrReplaceTransformer config = ConfigFactory.from_dict({ - 'regex_replace_tuple_list': [('a', 'b'), ('c', 'a')], - 'attribute_name': 'val' + REGEX_REPLACE_TUPLE_LIST: [('a', 'b'), ('c', 'a')], + ATTRIBUTE_NAME: 'val' }) transformer = RegexStrReplaceTransformer() @@ -46,6 +47,22 @@ def _default_test_transformer(self): return transformer + def test_dict_replace(self): + # type: () -> None + config = ConfigFactory.from_dict({ + REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')], + ATTRIBUTE_NAME: 'val' + }) + + transformer = RegexStrReplaceTransformer() + transformer.init(config) + + d = {'val': '\\'} + + actual = transformer.transform(d) + + self.assertEqual({'val': '\\\\'}, actual) + class Foo(object): def __init__(self, val):