From 35a510a405bbfe2b79db1f2d8b5fb62c5b773c4f Mon Sep 17 00:00:00 2001 From: Aaron White Date: Wed, 2 Mar 2022 11:36:23 -0500 Subject: [PATCH 1/4] fix: dremio sys.reflections schema changes based on version and reflection dataset lookup fails --- dbt/include/dremio/macros/adapters.sql | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/dbt/include/dremio/macros/adapters.sql b/dbt/include/dremio/macros/adapters.sql index 16bd9b8..dd8ed38 100644 --- a/dbt/include/dremio/macros/adapters.sql +++ b/dbt/include/dremio/macros/adapters.sql @@ -163,6 +163,19 @@ {% endmacro %} {% macro dremio__list_relations_without_caching(schema_relation) %} + {% set check_column_name_query %} + -- Dremio 22 uses "dataset_name" instead of "dataset" on sys.reflections + select + case when count(column_name) = 1 + then 'dataset' + else 'dataset_name' + end + from information_schema.columns + where table_schema = 'sys' + and table_name = 'reflections' + and column_name = 'dataset' + {% endset %} + {% set dataset_col = run_query(check_column_name_query).columns[0].values()[0] %} {% call statement('list_relations_without_caching', fetch_result=True) -%} with t1(table_catalog, table_name, table_schema, table_type) as ( select lower(case when position('.' in table_schema) > 0 @@ -179,15 +192,15 @@ ) ,r1(identifier_position, database_end_position, dataset, name, type) as ( select - case when "RIGHT"(dataset, 1) = '"' - then length(dataset) - strpos(substr(reverse(dataset), 2), '"') - else length(dataset) - strpos(reverse(dataset), '.') + 2 + case when "RIGHT"({{ dataset_col }}, 1) = '"' + then length({{ dataset_col }}) - strpos(substr(reverse({{ dataset_col }}), 2), '"') + else length({{ dataset_col }}) - strpos(reverse({{ dataset_col }}), '.') + 2 end - ,case when "LEFT"(dataset, 1) = '"' - then strpos(substr(dataset, 2), '"') + 1 - else strpos(dataset, '.') - 1 + ,case when "LEFT"({{ dataset_col }}, 1) = '"' + then strpos(substr({{ dataset_col }}, 2), '"') + 1 + else strpos({{ dataset_col }}, '.') - 1 end - ,dataset + ,{{ dataset_col }} ,name ,type from sys.reflections From 9952a17d3176e9842926f79aa01c382bccdf101f Mon Sep 17 00:00:00 2001 From: Aaron White Date: Wed, 2 Mar 2022 11:36:58 -0500 Subject: [PATCH 2/4] fix: reflection column name also changes based on dremio version --- dbt/include/dremio/macros/adapters.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/dremio/macros/adapters.sql b/dbt/include/dremio/macros/adapters.sql index dd8ed38..f53c139 100644 --- a/dbt/include/dremio/macros/adapters.sql +++ b/dbt/include/dremio/macros/adapters.sql @@ -176,6 +176,7 @@ and column_name = 'dataset' {% endset %} {% set dataset_col = run_query(check_column_name_query).columns[0].values()[0] %} + {% set reflection_col = 'name' if dataset_col == 'dataset' else 'reflection_name' %} {% call statement('list_relations_without_caching', fetch_result=True) -%} with t1(table_catalog, table_name, table_schema, table_type) as ( select lower(case when position('.' in table_schema) > 0 @@ -201,7 +202,7 @@ else strpos({{ dataset_col }}, '.') - 1 end ,{{ dataset_col }} - ,name + ,{{ reflection_col }} ,type from sys.reflections ) From 51087f2ce3e773e0311c13cb01e2ddb97d377806 Mon Sep 17 00:00:00 2001 From: Aaron White Date: Wed, 2 Mar 2022 19:32:16 -0500 Subject: [PATCH 3/4] fix: dremio version note is incorrect --- dbt/include/dremio/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/dremio/macros/adapters.sql b/dbt/include/dremio/macros/adapters.sql index f53c139..f6239ab 100644 --- a/dbt/include/dremio/macros/adapters.sql +++ b/dbt/include/dremio/macros/adapters.sql @@ -164,7 +164,7 @@ {% macro dremio__list_relations_without_caching(schema_relation) %} {% set check_column_name_query %} - -- Dremio 22 uses "dataset_name" instead of "dataset" on sys.reflections + -- Dremio 20 uses "dataset_name" instead of "dataset" on sys.reflections select case when count(column_name) = 1 then 'dataset' From f3cd821f416d9705c761be666e6b205f3a02e5ed Mon Sep 17 00:00:00 2001 From: Aaron White Date: Wed, 9 Mar 2022 14:56:33 -0500 Subject: [PATCH 4/4] fix: creating a reflection also needs to replace the column names --- dbt/include/dremio/macros/adapters.sql | 41 +++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/dbt/include/dremio/macros/adapters.sql b/dbt/include/dremio/macros/adapters.sql index f6239ab..1aa0542 100644 --- a/dbt/include/dremio/macros/adapters.sql +++ b/dbt/include/dremio/macros/adapters.sql @@ -27,7 +27,26 @@ ) {% endmacro %} +{% macro dremio__lookup_reflection_columns() -%} + {% set check_column_name_query %} + -- Dremio 20 uses "dataset_name" instead of "dataset" on sys.reflections + select + case when count(column_name) = 1 + then 'dataset' + else 'dataset_name' + end + from information_schema.columns + where table_schema = 'sys' + and table_name = 'reflections' + and column_name = 'dataset' + {% endset %} + {{ return(run_query(check_column_name_query).columns[0].values()[0]) }} +{%- endmacro %} + {% macro dremio__get_columns_in_relation(relation) -%} + {% set dataset_col = dremio__lookup_reflection_columns() %} + {% set reflection_col = 'reflection_name' if dataset_col == 'dataset_name' else 'name' %} + {% set display_col = 'display_columns' if dataset_col == 'dataset_name' else 'displayColumns' %} {% call statement('get_columns_in_relation', fetch_result=True) %} with cols as ( select lower(case when position('.' in table_schema) > 0 @@ -56,7 +75,7 @@ then substring(table_schema, position('.' in table_schema) + 1) else 'no_schema' end) - ,lower(name) + ,lower({{ reflection_col }}) ,lower(column_name) ,lower(data_type) ,character_maximum_length @@ -65,8 +84,8 @@ ,ordinal_position from sys.reflections join information_schema.columns - on (columns.table_schema || '.' || columns.table_name = replace(dataset, '"', '') - and (strpos(',' || replace(displayColumns, ' ', '') || ',', ',' || column_name || ',') > 0 + on (columns.table_schema || '.' || columns.table_name = replace({{ dataset_col }}, '"', '') + and (strpos(',' || replace({{ display_col }}, ' ', '') || ',', ',' || column_name || ',') > 0 or strpos(',' || replace(dimensions, ' ', '') || ',', ',' || column_name || ',') > 0 or strpos(',' || replace(measures, ' ', '') || ',', ',' || column_name || ',') > 0)) ) @@ -163,20 +182,8 @@ {% endmacro %} {% macro dremio__list_relations_without_caching(schema_relation) %} - {% set check_column_name_query %} - -- Dremio 20 uses "dataset_name" instead of "dataset" on sys.reflections - select - case when count(column_name) = 1 - then 'dataset' - else 'dataset_name' - end - from information_schema.columns - where table_schema = 'sys' - and table_name = 'reflections' - and column_name = 'dataset' - {% endset %} - {% set dataset_col = run_query(check_column_name_query).columns[0].values()[0] %} - {% set reflection_col = 'name' if dataset_col == 'dataset' else 'reflection_name' %} + {% set dataset_col = dremio__lookup_reflection_columns() %} + {% set reflection_col = 'reflection_name' if dataset_col == 'dataset_name' else 'name' %} {% call statement('list_relations_without_caching', fetch_result=True) -%} with t1(table_catalog, table_name, table_schema, table_type) as ( select lower(case when position('.' in table_schema) > 0