From 1c645e270b6b299983f034d6bf9b171808e5d2f9 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sat, 7 Oct 2023 20:05:49 +0100 Subject: [PATCH 01/15] feat: inspect columns --- dbt_project.yml | 3 ++ macros/insert_resources_from_graph.sql | 4 +- macros/unpack/get_column_values.sql | 46 +++++++++++++++++++ macros/unpack/get_resource_values.sql | 4 +- models/marts/core/int_all_columns.sql | 16 +++++++ .../staging/graph/base/base_node_columns.sql | 31 +++++++++++++ .../graph/base/base_source_columns.sql | 31 +++++++++++++ models/staging/graph/stg_columns.sql | 19 ++++++++ 8 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 macros/unpack/get_column_values.sql create mode 100644 models/marts/core/int_all_columns.sql create mode 100644 models/staging/graph/base/base_node_columns.sql create mode 100644 models/staging/graph/base/base_source_columns.sql create mode 100644 models/staging/graph/stg_columns.sql diff --git a/dbt_project.yml b/dbt_project.yml index 19a8b730..9e2c6b84 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -34,6 +34,9 @@ models: int_all_dag_relationships: # required for BigQuery, Redshift, and Databricks for performance/memory reasons +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}" + int_all_columns: + # required for BigQuery, Redshift, and Databricks for performance/memory reasons + +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}" dag: +materialized: table staging: diff --git a/macros/insert_resources_from_graph.sql b/macros/insert_resources_from_graph.sql index aeea9734..f20e9326 100644 --- a/macros/insert_resources_from_graph.sql +++ b/macros/insert_resources_from_graph.sql @@ -1,5 +1,5 @@ -{% macro insert_resources_from_graph(relation, resource_type='nodes', relationships=False, batch_size=var('insert_batch_size') | int) %} - {%- set values = get_resource_values(resource_type, relationships) -%} +{% macro insert_resources_from_graph(relation, resource_type='nodes', relationships=False, columns=False, batch_size=var('insert_batch_size') | int) %} + {%- set values = get_resource_values(resource_type, relationships, columns) -%} {%- set values_length = values | length -%} {%- set loop_count = (values_length / batch_size) | round(0, 'ceil') | int -%} diff --git a/macros/unpack/get_column_values.sql b/macros/unpack/get_column_values.sql new file mode 100644 index 00000000..69e6f601 --- /dev/null +++ b/macros/unpack/get_column_values.sql @@ -0,0 +1,46 @@ +{%- macro get_column_values(node_type) -%} + {{ return(adapter.dispatch('get_column_values', 'dbt_project_evaluator')(node_type)) }} +{%- endmacro -%} + +{%- macro default__get_column_values(node_type) -%} + + {%- if execute -%} + {%- if node_type == 'nodes' %} + {% set nodes_list = graph.nodes.values() %} + {%- elif node_type == 'sources' -%} + {% set nodes_list = graph.sources.values() %} + {%- else -%} + {{ exceptions.raise_compiler_error("node_type needs to be either nodes or sources, got " ~ node_type) }} + {% endif -%} + + {%- set values = [] -%} + + {%- for node in nodes_list -%} + {%- for column in node.columns.values() -%} + + {%- set values_line = + [ + wrap_string_with_quotes(node.unique_id), + wrap_string_with_quotes(node.name), + wrap_string_with_quotes(node.resource_type), + wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), + wrap_string_with_quotes(node.database), + wrap_string_with_quotes(node.schema), + wrap_string_with_quotes(node.package_name), + wrap_string_with_quotes(node.alias), + wrap_string_with_quotes(dbt.escape_single_quotes(column.name)), + wrap_string_with_quotes(dbt.escape_single_quotes(column.description)), + 'null' if not column.data_type else wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), + 'null' if not column.quote else wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) + ] + %} + + {%- do values.append(values_line) -%} + + {%- endfor -%} + {%- endfor -%} + {{ return(values) }} + + {%- endif -%} + +{%- endmacro -%} diff --git a/macros/unpack/get_resource_values.sql b/macros/unpack/get_resource_values.sql index 47c301f8..4fe76d59 100644 --- a/macros/unpack/get_resource_values.sql +++ b/macros/unpack/get_resource_values.sql @@ -1,6 +1,8 @@ -{% macro get_resource_values(resource=None, relationships=None) %} +{% macro get_resource_values(resource=None, relationships=None, columns=None) %} {% if relationships %} {{ return(adapter.dispatch('get_relationship_values', 'dbt_project_evaluator')(node_type=resource)) }} + {% elif columns %} + {{ return(adapter.dispatch('get_column_values', 'dbt_project_evaluator')(node_type=resource)) }} {% elif resource == 'exposures' %} {{ return(adapter.dispatch('get_exposure_values', 'dbt_project_evaluator')()) }} {% elif resource == 'sources' %} diff --git a/models/marts/core/int_all_columns.sql b/models/marts/core/int_all_columns.sql new file mode 100644 index 00000000..671f2338 --- /dev/null +++ b/models/marts/core/int_all_columns.sql @@ -0,0 +1,16 @@ +-- one row for each column in a node or source + +select + columns.unique_id, + columns.node_name, + columns.resource_type, + columns.file_path, + columns.database, + columns.schema, + columns.package_name, + columns.alias, + columns.name, + columns.description, + columns.data_type, + columns.quote +from {{ ref('stg_columns') }} as columns diff --git a/models/staging/graph/base/base_node_columns.sql b/models/staging/graph/base/base_node_columns.sql new file mode 100644 index 00000000..529bd4c2 --- /dev/null +++ b/models/staging/graph/base/base_node_columns.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized='table', + post_hook="{{ insert_resources_from_graph(this, resource_type='nodes', columns=True) }}" + ) +}} + +{% if execute %} + {{ check_model_is_table(model) }} +{% endif %} +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ dbt.type_string() }}) as unique_id, + cast(null as {{ dbt.type_string() }}) as node_name, + cast(null as {{ dbt.type_string() }}) as resource_type, + cast(null as {{ dbt.type_string() }}) as file_path, + cast(null as {{ dbt.type_string() }}) as database, + cast(null as {{ dbt.type_string() }}) as schema, + cast(null as {{ dbt.type_string() }}) as package_name, + cast(null as {{ dbt.type_string() }}) as alias, + cast(null as {{ dbt.type_string()}}) as name, + cast(null as {{ dbt.type_string()}}) as description, + cast(null as {{ dbt.type_string()}}) as data_type, + cast(null as {{ dbt.type_string()}}) as quote + +from dummy_cte +where false \ No newline at end of file diff --git a/models/staging/graph/base/base_source_columns.sql b/models/staging/graph/base/base_source_columns.sql new file mode 100644 index 00000000..7ea662a9 --- /dev/null +++ b/models/staging/graph/base/base_source_columns.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized='table', + post_hook="{{ insert_resources_from_graph(this, resource_type='sources', columns=True) }}" + ) +}} + +{% if execute %} + {{ check_model_is_table(model) }} +{% endif %} +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ dbt.type_string() }}) as unique_id, + cast(null as {{ dbt.type_string() }}) as node_name, + cast(null as {{ dbt.type_string() }}) as resource_type, + cast(null as {{ dbt.type_string() }}) as file_path, + cast(null as {{ dbt.type_string() }}) as database, + cast(null as {{ dbt.type_string() }}) as schema, + cast(null as {{ dbt.type_string() }}) as package_name, + cast(null as {{ dbt.type_string() }}) as alias, + cast(null as {{ dbt.type_string()}}) as name, + cast(null as {{ dbt.type_string()}}) as description, + cast(null as {{ dbt.type_string()}}) as data_type, + cast(False as boolean) as quote + +from dummy_cte +where false \ No newline at end of file diff --git a/models/staging/graph/stg_columns.sql b/models/staging/graph/stg_columns.sql new file mode 100644 index 00000000..cefe3f4f --- /dev/null +++ b/models/staging/graph/stg_columns.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='table', + ) +}} + +{% if execute %} + {{ check_model_is_table(model) }} +{% endif %} + +with unioned as ( + + {{ dbt_utils.union_relations([ + ref('base_node_columns'), + ref('base_source_columns') + ])}} +) + +select distinct * from unioned \ No newline at end of file From 860eedb611e719545205bac39f173bbad68a7297 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Tue, 24 Oct 2023 22:49:38 +0100 Subject: [PATCH 02/15] Describe new models in graph and core yaml files --- .../dbt_project_evaluator_schema_tests/core.yml | 12 ++++++++++-- .../dbt_project_evaluator_schema_tests/graph.yml | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml index 5f0f2af7..463698d9 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml @@ -9,7 +9,7 @@ models: tests: - unique - not_null - + - name: int_all_graph_resources description: "This table shows one record for each enabled resource in the graph and information about that resource." columns: @@ -28,4 +28,12 @@ models: - name: unique_id tests: - unique - - not_null \ No newline at end of file + - not_null + + - name: int_all_columns + description: "This table shows one record for each column for all nodes and sources in the graph." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - unique_id + - column_name diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml index 739d321c..df30e518 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml @@ -49,6 +49,14 @@ models: - unique - not_null + - name: stg_columns + description: "Staging model from the graph variable, one record per column resource." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - unique_id + - column_name + - name: stg_sources description: "Staging model from the graph variable, one record per source resource." columns: From 4a327b2f69b958c44699398738ff8e82b6e3125d Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Tue, 24 Oct 2023 23:03:51 +0100 Subject: [PATCH 03/15] Add a new docs page 'Querying Columns' --- docs/customization/querying-columns.md | 16 ++++++++++++++++ .../dbt_project_evaluator_schema_tests/core.yml | 2 +- .../dbt_project_evaluator_schema_tests/graph.yml | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 docs/customization/querying-columns.md diff --git a/docs/customization/querying-columns.md b/docs/customization/querying-columns.md new file mode 100644 index 00000000..4e730374 --- /dev/null +++ b/docs/customization/querying-columns.md @@ -0,0 +1,16 @@ +# Querying columns with SQL + +The model `int_all_columns` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/core/int_all_columns.sql)), created with the package, lists all the columns from all the dbt nodes (models, sources, seeds, snapshots, analyses) + +You can use this model to help with questions such as: + +- Are there columns with the same name in different nodes? +- Are there columns that lack descriptions? +- Do any columns share the same name but have different descriptions? +- Are there columns with names that match a specific pattern (regex)? +- Have any prohibited names been used for columns? + + +## Defining additional tests that match your exact requirements + +- writing a model with `ref(int_all_columns)` with custom tests added for a specific use case diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml index 463698d9..fa05933b 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml @@ -36,4 +36,4 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - unique_id - - column_name + - name diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml index df30e518..05f1b10a 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml @@ -55,7 +55,7 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - unique_id - - column_name + - name - name: stg_sources description: "Staging model from the graph variable, one record per source resource." From cd7d880002100d15406d50937799e4e1eae729d3 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Tue, 24 Oct 2023 23:15:35 +0100 Subject: [PATCH 04/15] Update mkdocs to link new page --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index ed205fa5..4679af57 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - Configuring exceptions to the rules: customization/exceptions.md - Excluding packages and models/sources based on path: customization/excluding-packages-and-paths.md - Display issues in the logs: customization/issues-in-log.md + - Querying columns: customization/querying-columns.md - Run in CI Check: ci-check.md - Querying the DAG: querying-the-dag.md - Contributing: contributing.md \ No newline at end of file From 4edf16bc53ce3a19cd98a0b162a80cd8f47e9361 Mon Sep 17 00:00:00 2001 From: gastlich Date: Sun, 29 Oct 2023 12:44:32 +0000 Subject: [PATCH 05/15] Update docs/customization/querying-columns.md Co-authored-by: dave-connors-3 <73915542+dave-connors-3@users.noreply.github.com> --- docs/customization/querying-columns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/customization/querying-columns.md b/docs/customization/querying-columns.md index 4e730374..eeb94de0 100644 --- a/docs/customization/querying-columns.md +++ b/docs/customization/querying-columns.md @@ -13,4 +13,4 @@ You can use this model to help with questions such as: ## Defining additional tests that match your exact requirements -- writing a model with `ref(int_all_columns)` with custom tests added for a specific use case +You can create a custom test against `{{ ref(int_all_columns) }}` to test for your specific check! When running the package you'd need to make sure to also include children of the packag's models by using the `package:dbt_project_evalutator+` selector. From 53c6fb6c60ab7300788dd8d323550de3a57ba03d Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sun, 29 Oct 2023 13:22:14 +0000 Subject: [PATCH 06/15] import CTE and join with stg_nodes --- .../core.yml | 2 +- .../graph.yml | 2 +- macros/unpack/get_column_values.sql | 7 -- models/marts/core/int_all_columns.sql | 71 +++++++++++++++---- .../staging/graph/base/base_node_columns.sql | 9 +-- .../graph/base/base_source_columns.sql | 11 +-- 6 files changed, 62 insertions(+), 40 deletions(-) diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml index fa05933b..707d338a 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml @@ -35,5 +35,5 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - unique_id + - node_unique_id - name diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml index 05f1b10a..afcab7f9 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/graph.yml @@ -54,7 +54,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - unique_id + - node_unique_id - name - name: stg_sources diff --git a/macros/unpack/get_column_values.sql b/macros/unpack/get_column_values.sql index 69e6f601..e7aa4ed2 100644 --- a/macros/unpack/get_column_values.sql +++ b/macros/unpack/get_column_values.sql @@ -21,13 +21,6 @@ {%- set values_line = [ wrap_string_with_quotes(node.unique_id), - wrap_string_with_quotes(node.name), - wrap_string_with_quotes(node.resource_type), - wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), - wrap_string_with_quotes(node.database), - wrap_string_with_quotes(node.schema), - wrap_string_with_quotes(node.package_name), - wrap_string_with_quotes(node.alias), wrap_string_with_quotes(dbt.escape_single_quotes(column.name)), wrap_string_with_quotes(dbt.escape_single_quotes(column.description)), 'null' if not column.data_type else wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), diff --git a/models/marts/core/int_all_columns.sql b/models/marts/core/int_all_columns.sql index 671f2338..090b96ef 100644 --- a/models/marts/core/int_all_columns.sql +++ b/models/marts/core/int_all_columns.sql @@ -1,16 +1,59 @@ -- one row for each column in a node or source -select - columns.unique_id, - columns.node_name, - columns.resource_type, - columns.file_path, - columns.database, - columns.schema, - columns.package_name, - columns.alias, - columns.name, - columns.description, - columns.data_type, - columns.quote -from {{ ref('stg_columns') }} as columns +with + +stg_columns as ( + + select + node_unique_id, + name, + description, + data_type, + quote + + from {{ ref('stg_columns') }} + +), + +stg_nodes as ( + + select + unique_id, + name, + resource_type, + file_path, + is_enabled, + database, + schema, + package_name, + alias + + from {{ ref('stg_nodes') }} + +), + +final as ( + + select + stg_nodes.unique_id as node_unique_id, + stg_nodes.name as node_name, + stg_nodes.resource_type as node_resource_type, + stg_nodes.file_path as node_file_path, + stg_nodes.is_enabled as node_is_enabled, + stg_nodes.database as node_database, + stg_nodes.schema as node_schema, + stg_nodes.package_name as node_package_name, + stg_nodes.alias as node_alias, + stg_columns.name as name, + stg_columns.description as description, + stg_columns.data_type as data_type, + stg_columns.quote as quote + + from stg_columns + + right join stg_nodes + on stg_nodes.unique_id = stg_columns.node_unique_id + +) + +select * from final diff --git a/models/staging/graph/base/base_node_columns.sql b/models/staging/graph/base/base_node_columns.sql index 529bd4c2..c6297cf9 100644 --- a/models/staging/graph/base/base_node_columns.sql +++ b/models/staging/graph/base/base_node_columns.sql @@ -14,14 +14,7 @@ with dummy_cte as ( ) select - cast(null as {{ dbt.type_string() }}) as unique_id, - cast(null as {{ dbt.type_string() }}) as node_name, - cast(null as {{ dbt.type_string() }}) as resource_type, - cast(null as {{ dbt.type_string() }}) as file_path, - cast(null as {{ dbt.type_string() }}) as database, - cast(null as {{ dbt.type_string() }}) as schema, - cast(null as {{ dbt.type_string() }}) as package_name, - cast(null as {{ dbt.type_string() }}) as alias, + cast(null as {{ dbt.type_string() }}) as node_unique_id, cast(null as {{ dbt.type_string()}}) as name, cast(null as {{ dbt.type_string()}}) as description, cast(null as {{ dbt.type_string()}}) as data_type, diff --git a/models/staging/graph/base/base_source_columns.sql b/models/staging/graph/base/base_source_columns.sql index 7ea662a9..4592ecf5 100644 --- a/models/staging/graph/base/base_source_columns.sql +++ b/models/staging/graph/base/base_source_columns.sql @@ -14,18 +14,11 @@ with dummy_cte as ( ) select - cast(null as {{ dbt.type_string() }}) as unique_id, - cast(null as {{ dbt.type_string() }}) as node_name, - cast(null as {{ dbt.type_string() }}) as resource_type, - cast(null as {{ dbt.type_string() }}) as file_path, - cast(null as {{ dbt.type_string() }}) as database, - cast(null as {{ dbt.type_string() }}) as schema, - cast(null as {{ dbt.type_string() }}) as package_name, - cast(null as {{ dbt.type_string() }}) as alias, + cast(null as {{ dbt.type_string() }}) as node_unique_id, cast(null as {{ dbt.type_string()}}) as name, cast(null as {{ dbt.type_string()}}) as description, cast(null as {{ dbt.type_string()}}) as data_type, - cast(False as boolean) as quote + cast(null as {{ dbt.type_string()}}) as quote from dummy_cte where false \ No newline at end of file From dcd1cfccbab8991ca18c4aa520244de63a4da767 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sun, 29 Oct 2023 13:31:57 +0000 Subject: [PATCH 07/15] Don't materialise stg_columns --- models/staging/graph/stg_columns.sql | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/models/staging/graph/stg_columns.sql b/models/staging/graph/stg_columns.sql index cefe3f4f..52dd8057 100644 --- a/models/staging/graph/stg_columns.sql +++ b/models/staging/graph/stg_columns.sql @@ -1,14 +1,6 @@ -{{ - config( - materialized='table', - ) -}} +with -{% if execute %} - {{ check_model_is_table(model) }} -{% endif %} - -with unioned as ( +final as ( {{ dbt_utils.union_relations([ ref('base_node_columns'), @@ -16,4 +8,4 @@ with unioned as ( ])}} ) -select distinct * from unioned \ No newline at end of file +select distinct * from final From e3c1afe24347f5d6882e1b96360e3de733e771fa Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sun, 29 Oct 2023 13:53:23 +0000 Subject: [PATCH 08/15] Improve description for querying columns page --- docs/customization/querying-columns.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/customization/querying-columns.md b/docs/customization/querying-columns.md index eeb94de0..5eee8f44 100644 --- a/docs/customization/querying-columns.md +++ b/docs/customization/querying-columns.md @@ -1,11 +1,11 @@ # Querying columns with SQL -The model `int_all_columns` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/core/int_all_columns.sql)), created with the package, lists all the columns from all the dbt nodes (models, sources, seeds, snapshots, analyses) +The model `int_all_columns` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/core/int_all_columns.sql)), created with the package, lists all the columns from all the dbt nodes (models, sources, tests, snapshots) You can use this model to help with questions such as: - Are there columns with the same name in different nodes? -- Are there columns that lack descriptions? +- Do any columns in the YAML configuration lack descriptions? - Do any columns share the same name but have different descriptions? - Are there columns with names that match a specific pattern (regex)? - Have any prohibited names been used for columns? From 141417d2825cb9ef57a66821a826705995f04eb9 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sun, 5 Nov 2023 23:25:36 +0000 Subject: [PATCH 09/15] Remove distinct clause from stg_columns --- models/staging/graph/stg_columns.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/graph/stg_columns.sql b/models/staging/graph/stg_columns.sql index 52dd8057..8fd8df3a 100644 --- a/models/staging/graph/stg_columns.sql +++ b/models/staging/graph/stg_columns.sql @@ -8,4 +8,4 @@ final as ( ])}} ) -select distinct * from final +select * from final From 0bfa709d93c9d7366b8506d001adcf978c26bc35 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sun, 5 Nov 2023 23:27:31 +0000 Subject: [PATCH 10/15] Inner join with stg_nodes instead of right join --- models/marts/core/int_all_columns.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/int_all_columns.sql b/models/marts/core/int_all_columns.sql index 090b96ef..cae3112e 100644 --- a/models/marts/core/int_all_columns.sql +++ b/models/marts/core/int_all_columns.sql @@ -51,7 +51,7 @@ final as ( from stg_columns - right join stg_nodes + inner join stg_nodes on stg_nodes.unique_id = stg_columns.node_unique_id ) From 0a69c5144fb75b6cbf8f31ce3bdb3f5ec5f27cf9 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Fri, 5 Jan 2024 11:16:13 +0000 Subject: [PATCH 11/15] Add integrations tests for int_all_columns model --- integration_tests/seeds/core/core_seeds.yml | 24 +++++++++++++ .../seeds/core/test_int_all_columns.csv | 35 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 integration_tests/seeds/core/core_seeds.yml create mode 100644 integration_tests/seeds/core/test_int_all_columns.csv diff --git a/integration_tests/seeds/core/core_seeds.yml b/integration_tests/seeds/core/core_seeds.yml new file mode 100644 index 00000000..9b44042e --- /dev/null +++ b/integration_tests/seeds/core/core_seeds.yml @@ -0,0 +1,24 @@ +version: 2 + +seeds: + - name: test_int_all_columns + config: + column_types: + quote: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino'] else 'string' }}" + tests: + - dbt_utils.equality: + name: equality_int_all_columns + compare_model: ref('int_all_columns') + compare_columns: + - node_unique_id + - node_name + - node_resource_type + - node_file_path + - node_is_enabled + - node_schema + - node_package_name + - node_alias + - name + - description + - data_type + - quote diff --git a/integration_tests/seeds/core/test_int_all_columns.csv b/integration_tests/seeds/core/test_int_all_columns.csv new file mode 100644 index 00000000..d6df6507 --- /dev/null +++ b/integration_tests/seeds/core/test_int_all_columns.csv @@ -0,0 +1,35 @@ +node_unique_id,node_name,node_resource_type,node_file_path,node_is_enabled,node_database,node_schema,node_package_name,node_alias,name,description,data_type,quote +model.exclude_package.excluded_model,excluded_model,model,models/staging/excluded_model.sql,true,database,integration_tests,exclude_package,excluded_model,id,,integer, +model.dbt_project_evaluator.stg_nodes,stg_nodes,model,models/staging/graph/stg_nodes.sql,true,database,integration_tests,dbt_project_evaluator,stg_nodes,unique_id,,, +model.dbt_project_evaluator.stg_metrics,stg_metrics,model,models/staging/graph/stg_metrics.sql,true,database,integration_tests,dbt_project_evaluator,stg_metrics,unique_id,,, +model.dbt_project_evaluator.stg_sources,stg_sources,model,models/staging/graph/stg_sources.sql,true,database,integration_tests,dbt_project_evaluator,stg_sources,unique_id,,, +model.dbt_project_evaluator.stg_exposures,stg_exposures,model,models/staging/graph/stg_exposures.sql,true,database,integration_tests,dbt_project_evaluator,stg_exposures,unique_id,,, +model.dbt_project_evaluator.fct_test_coverage,fct_test_coverage,model,models/marts/tests/fct_test_coverage.sql,true,database,integration_tests,dbt_project_evaluator,fct_test_coverage,test_to_model_ratio,the total number of tests in the project divided by the total number of models in the project,, +model.dbt_project_evaluator.fct_test_coverage,fct_test_coverage,model,models/marts/tests/fct_test_coverage.sql,true,database,integration_tests,dbt_project_evaluator,fct_test_coverage,test_coverage_pct,the number of models in the project with at least one test configured divided by the total number of models in the project,, +model.dbt_project_evaluator.int_all_graph_resources,int_all_graph_resources,model,models/marts/core/int_all_graph_resources.sql,true,database,integration_tests,dbt_project_evaluator,int_all_graph_resources,resource_id,,, +model.dbt_project_evaluator.int_all_graph_resources,int_all_graph_resources,model,models/marts/core/int_all_graph_resources.sql,true,database,integration_tests,dbt_project_evaluator,int_all_graph_resources,on_schema_change,"this contains the on_schema_change setting for incremental models. This column was sometimes an empty string, so should be tested to detect regressions",, +model.dbt_project_evaluator.int_all_graph_resources,int_all_graph_resources,model,models/marts/core/int_all_graph_resources.sql,true,database,integration_tests,dbt_project_evaluator,int_all_graph_resources,model_type,,, +model.dbt_project_evaluator.int_direct_relationships,int_direct_relationships,model,models/marts/core/int_direct_relationships.sql,true,database,integration_tests,dbt_project_evaluator,int_direct_relationships,unique_id,,, +model.dbt_project_evaluator.int_all_dag_relationships,int_all_dag_relationships,model,models/marts/core/int_all_dag_relationships.sql,true,database,integration_tests,dbt_project_evaluator,int_all_dag_relationships,path,unique identifier representing a distinct path from each resource to another resource,, +model.dbt_project_evaluator.stg_node_relationships,stg_node_relationships,model,models/staging/graph/stg_node_relationships.sql,true,database,integration_tests,dbt_project_evaluator,stg_node_relationships,unique_id,,, +model.dbt_project_evaluator.stg_metric_relationships,stg_metric_relationships,model,models/staging/graph/stg_metric_relationships.sql,true,database,integration_tests,dbt_project_evaluator,stg_metric_relationships,unique_id,,, +model.dbt_project_evaluator.stg_exposure_relationships,stg_exposure_relationships,model,models/staging/graph/stg_exposure_relationships.sql,true,database,integration_tests,dbt_project_evaluator,stg_exposure_relationships,unique_id,,, +model.dbt_project_evaluator.fct_documentation_coverage,fct_documentation_coverage,model,models/marts/documentation/fct_documentation_coverage.sql,true,database,integration_tests,dbt_project_evaluator,fct_documentation_coverage,documentation_coverage_pct,the number of models in the project with a description divided by the total number of models in the project,, +model.dbt_project_evaluator.int_model_test_summary,int_model_test_summary,model,models/marts/tests/intermediate/int_model_test_summary.sql,true,database,integration_tests,dbt_project_evaluator,int_model_test_summary,resource_name,,, +model.dbt_project_evaluator.stg_naming_convention_folders,stg_naming_convention_folders,model,models/staging/variables/stg_naming_convention_folders.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_folders,model_type,,, +model.dbt_project_evaluator.stg_naming_convention_folders,stg_naming_convention_folders,model,models/staging/variables/stg_naming_convention_folders.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_folders,folder_name,,, +model.dbt_project_evaluator.fct_chained_views_dependencies,fct_chained_views_dependencies,model,models/marts/performance/fct_chained_views_dependencies.sql,true,database,integration_tests,dbt_project_evaluator,fct_chained_views_dependencies,distance,,, +model.dbt_project_evaluator.stg_naming_convention_prefixes,stg_naming_convention_prefixes,model,models/staging/variables/stg_naming_convention_prefixes.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_prefixes,model_type,,, +model.dbt_project_evaluator.stg_naming_convention_prefixes,stg_naming_convention_prefixes,model,models/staging/variables/stg_naming_convention_prefixes.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_prefixes,unique_id,surrogate key of model_type and prefix value,, +model.dbt_project_evaluator_integration_tests.report_1,report_1,model,models/reports/report_1.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,report_1,col,col me maybe,boolean, +model.dbt_project_evaluator_integration_tests.report_2,report_2,model,models/reports/report_2.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,report_2,id,,integer, +model.dbt_project_evaluator_integration_tests.int_model_4,int_model_4,model,models/marts/int_model_4.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,int_model_4,id,,, +model.dbt_project_evaluator_integration_tests.stg_model_4,stg_model_4,model,models/staging/stg_model_4.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_4,id,,, +model.dbt_project_evaluator_integration_tests.stg_model_4,stg_model_4,model,models/staging/stg_model_4.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_4,"concat(coalesce('fake_column', ' '),'for_testing')",,, +model.dbt_project_evaluator_integration_tests.metricflow_time_spine,metricflow_time_spine,model,models/utils/metricflow_time_spine.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,metricflow_time_spine,date_day,,, +model.dbt_project_evaluator_integration_tests.stg_model_1,stg_model_1,model,models/staging/source_1/stg_model_1.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_1,id,hocus pocus,, +model.dbt_project_evaluator_integration_tests.stg_model_2,stg_model_2,model,models/staging/source_1/stg_model_2.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_2,id,,, +seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,id_to_exclude,Values (or like pattern) to exclude for column_name.,, +seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,comment,Field to document why a given exception is legitimate.,, +seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,fct_name,Name of the fact table to define exceptions.,, +seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,column_name,Column name from fct_name to define exceptions.,, \ No newline at end of file From 3556df76610ea3e665c085394e8c2fb556fb4f1e Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Sun, 21 Jan 2024 15:16:08 +0000 Subject: [PATCH 12/15] Remove unneeded equality tests --- integration_tests/seeds/core/core_seeds.yml | 24 ------------- .../seeds/core/test_int_all_columns.csv | 35 ------------------- 2 files changed, 59 deletions(-) delete mode 100644 integration_tests/seeds/core/core_seeds.yml delete mode 100644 integration_tests/seeds/core/test_int_all_columns.csv diff --git a/integration_tests/seeds/core/core_seeds.yml b/integration_tests/seeds/core/core_seeds.yml deleted file mode 100644 index 9b44042e..00000000 --- a/integration_tests/seeds/core/core_seeds.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -seeds: - - name: test_int_all_columns - config: - column_types: - quote: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino'] else 'string' }}" - tests: - - dbt_utils.equality: - name: equality_int_all_columns - compare_model: ref('int_all_columns') - compare_columns: - - node_unique_id - - node_name - - node_resource_type - - node_file_path - - node_is_enabled - - node_schema - - node_package_name - - node_alias - - name - - description - - data_type - - quote diff --git a/integration_tests/seeds/core/test_int_all_columns.csv b/integration_tests/seeds/core/test_int_all_columns.csv deleted file mode 100644 index d6df6507..00000000 --- a/integration_tests/seeds/core/test_int_all_columns.csv +++ /dev/null @@ -1,35 +0,0 @@ -node_unique_id,node_name,node_resource_type,node_file_path,node_is_enabled,node_database,node_schema,node_package_name,node_alias,name,description,data_type,quote -model.exclude_package.excluded_model,excluded_model,model,models/staging/excluded_model.sql,true,database,integration_tests,exclude_package,excluded_model,id,,integer, -model.dbt_project_evaluator.stg_nodes,stg_nodes,model,models/staging/graph/stg_nodes.sql,true,database,integration_tests,dbt_project_evaluator,stg_nodes,unique_id,,, -model.dbt_project_evaluator.stg_metrics,stg_metrics,model,models/staging/graph/stg_metrics.sql,true,database,integration_tests,dbt_project_evaluator,stg_metrics,unique_id,,, -model.dbt_project_evaluator.stg_sources,stg_sources,model,models/staging/graph/stg_sources.sql,true,database,integration_tests,dbt_project_evaluator,stg_sources,unique_id,,, -model.dbt_project_evaluator.stg_exposures,stg_exposures,model,models/staging/graph/stg_exposures.sql,true,database,integration_tests,dbt_project_evaluator,stg_exposures,unique_id,,, -model.dbt_project_evaluator.fct_test_coverage,fct_test_coverage,model,models/marts/tests/fct_test_coverage.sql,true,database,integration_tests,dbt_project_evaluator,fct_test_coverage,test_to_model_ratio,the total number of tests in the project divided by the total number of models in the project,, -model.dbt_project_evaluator.fct_test_coverage,fct_test_coverage,model,models/marts/tests/fct_test_coverage.sql,true,database,integration_tests,dbt_project_evaluator,fct_test_coverage,test_coverage_pct,the number of models in the project with at least one test configured divided by the total number of models in the project,, -model.dbt_project_evaluator.int_all_graph_resources,int_all_graph_resources,model,models/marts/core/int_all_graph_resources.sql,true,database,integration_tests,dbt_project_evaluator,int_all_graph_resources,resource_id,,, -model.dbt_project_evaluator.int_all_graph_resources,int_all_graph_resources,model,models/marts/core/int_all_graph_resources.sql,true,database,integration_tests,dbt_project_evaluator,int_all_graph_resources,on_schema_change,"this contains the on_schema_change setting for incremental models. This column was sometimes an empty string, so should be tested to detect regressions",, -model.dbt_project_evaluator.int_all_graph_resources,int_all_graph_resources,model,models/marts/core/int_all_graph_resources.sql,true,database,integration_tests,dbt_project_evaluator,int_all_graph_resources,model_type,,, -model.dbt_project_evaluator.int_direct_relationships,int_direct_relationships,model,models/marts/core/int_direct_relationships.sql,true,database,integration_tests,dbt_project_evaluator,int_direct_relationships,unique_id,,, -model.dbt_project_evaluator.int_all_dag_relationships,int_all_dag_relationships,model,models/marts/core/int_all_dag_relationships.sql,true,database,integration_tests,dbt_project_evaluator,int_all_dag_relationships,path,unique identifier representing a distinct path from each resource to another resource,, -model.dbt_project_evaluator.stg_node_relationships,stg_node_relationships,model,models/staging/graph/stg_node_relationships.sql,true,database,integration_tests,dbt_project_evaluator,stg_node_relationships,unique_id,,, -model.dbt_project_evaluator.stg_metric_relationships,stg_metric_relationships,model,models/staging/graph/stg_metric_relationships.sql,true,database,integration_tests,dbt_project_evaluator,stg_metric_relationships,unique_id,,, -model.dbt_project_evaluator.stg_exposure_relationships,stg_exposure_relationships,model,models/staging/graph/stg_exposure_relationships.sql,true,database,integration_tests,dbt_project_evaluator,stg_exposure_relationships,unique_id,,, -model.dbt_project_evaluator.fct_documentation_coverage,fct_documentation_coverage,model,models/marts/documentation/fct_documentation_coverage.sql,true,database,integration_tests,dbt_project_evaluator,fct_documentation_coverage,documentation_coverage_pct,the number of models in the project with a description divided by the total number of models in the project,, -model.dbt_project_evaluator.int_model_test_summary,int_model_test_summary,model,models/marts/tests/intermediate/int_model_test_summary.sql,true,database,integration_tests,dbt_project_evaluator,int_model_test_summary,resource_name,,, -model.dbt_project_evaluator.stg_naming_convention_folders,stg_naming_convention_folders,model,models/staging/variables/stg_naming_convention_folders.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_folders,model_type,,, -model.dbt_project_evaluator.stg_naming_convention_folders,stg_naming_convention_folders,model,models/staging/variables/stg_naming_convention_folders.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_folders,folder_name,,, -model.dbt_project_evaluator.fct_chained_views_dependencies,fct_chained_views_dependencies,model,models/marts/performance/fct_chained_views_dependencies.sql,true,database,integration_tests,dbt_project_evaluator,fct_chained_views_dependencies,distance,,, -model.dbt_project_evaluator.stg_naming_convention_prefixes,stg_naming_convention_prefixes,model,models/staging/variables/stg_naming_convention_prefixes.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_prefixes,model_type,,, -model.dbt_project_evaluator.stg_naming_convention_prefixes,stg_naming_convention_prefixes,model,models/staging/variables/stg_naming_convention_prefixes.sql,true,database,integration_tests,dbt_project_evaluator,stg_naming_convention_prefixes,unique_id,surrogate key of model_type and prefix value,, -model.dbt_project_evaluator_integration_tests.report_1,report_1,model,models/reports/report_1.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,report_1,col,col me maybe,boolean, -model.dbt_project_evaluator_integration_tests.report_2,report_2,model,models/reports/report_2.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,report_2,id,,integer, -model.dbt_project_evaluator_integration_tests.int_model_4,int_model_4,model,models/marts/int_model_4.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,int_model_4,id,,, -model.dbt_project_evaluator_integration_tests.stg_model_4,stg_model_4,model,models/staging/stg_model_4.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_4,id,,, -model.dbt_project_evaluator_integration_tests.stg_model_4,stg_model_4,model,models/staging/stg_model_4.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_4,"concat(coalesce('fake_column', ' '),'for_testing')",,, -model.dbt_project_evaluator_integration_tests.metricflow_time_spine,metricflow_time_spine,model,models/utils/metricflow_time_spine.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,metricflow_time_spine,date_day,,, -model.dbt_project_evaluator_integration_tests.stg_model_1,stg_model_1,model,models/staging/source_1/stg_model_1.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_1,id,hocus pocus,, -model.dbt_project_evaluator_integration_tests.stg_model_2,stg_model_2,model,models/staging/source_1/stg_model_2.sql,true,database,integration_tests,dbt_project_evaluator_integration_tests,stg_model_2,id,,, -seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,id_to_exclude,Values (or like pattern) to exclude for column_name.,, -seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,comment,Field to document why a given exception is legitimate.,, -seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,fct_name,Name of the fact table to define exceptions.,, -seed.dbt_project_evaluator_integration_tests.dbt_project_evaluator_exceptions,dbt_project_evaluator_exceptions,seed,seeds/dbt_project_evaluator_exceptions.csv,true,database,integration_tests,dbt_project_evaluator_integration_tests,dbt_project_evaluator_exceptions,column_name,Column name from fct_name to define exceptions.,, \ No newline at end of file From df44df7f2a29e67bf9f57af81a1c1024bea863d6 Mon Sep 17 00:00:00 2001 From: Jacek Wojna Date: Wed, 24 Jan 2024 08:34:13 +0000 Subject: [PATCH 13/15] Remove int_all_columns model --- dbt_project.yml | 3 - docs/customization/querying-columns.md | 4 +- .../core.yml | 8 --- models/marts/core/int_all_columns.sql | 59 ------------------- 4 files changed, 2 insertions(+), 72 deletions(-) delete mode 100644 models/marts/core/int_all_columns.sql diff --git a/dbt_project.yml b/dbt_project.yml index f53cbfa3..e706fa5d 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -34,9 +34,6 @@ models: int_all_dag_relationships: # required for BigQuery, Redshift, and Databricks for performance/memory reasons +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}" - int_all_columns: - # required for BigQuery, Redshift, and Databricks for performance/memory reasons - +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}" dag: +materialized: table staging: diff --git a/docs/customization/querying-columns.md b/docs/customization/querying-columns.md index 5eee8f44..31289d40 100644 --- a/docs/customization/querying-columns.md +++ b/docs/customization/querying-columns.md @@ -1,6 +1,6 @@ # Querying columns with SQL -The model `int_all_columns` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/core/int_all_columns.sql)), created with the package, lists all the columns from all the dbt nodes (models, sources, tests, snapshots) +The model `stg_columns` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/staging/graph/stg_columns.sql)), created with the package, lists all the columns from all the dbt nodes (models, sources, tests, snapshots) You can use this model to help with questions such as: @@ -13,4 +13,4 @@ You can use this model to help with questions such as: ## Defining additional tests that match your exact requirements -You can create a custom test against `{{ ref(int_all_columns) }}` to test for your specific check! When running the package you'd need to make sure to also include children of the packag's models by using the `package:dbt_project_evalutator+` selector. +You can create a custom test against `{{ ref(stg_columns) }}` to test for your specific check! When running the package you'd need to make sure to also include children of the package's models by using the `package:dbt_project_evalutator+` selector. diff --git a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml index 3bd4e1a8..6865799b 100644 --- a/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml +++ b/integration_tests/models/dbt_project_evaluator_schema_tests/core.yml @@ -35,11 +35,3 @@ models: tests: - unique - not_null - - - name: int_all_columns - description: "This table shows one record for each column for all nodes and sources in the graph." - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - node_unique_id - - name diff --git a/models/marts/core/int_all_columns.sql b/models/marts/core/int_all_columns.sql deleted file mode 100644 index cae3112e..00000000 --- a/models/marts/core/int_all_columns.sql +++ /dev/null @@ -1,59 +0,0 @@ --- one row for each column in a node or source - -with - -stg_columns as ( - - select - node_unique_id, - name, - description, - data_type, - quote - - from {{ ref('stg_columns') }} - -), - -stg_nodes as ( - - select - unique_id, - name, - resource_type, - file_path, - is_enabled, - database, - schema, - package_name, - alias - - from {{ ref('stg_nodes') }} - -), - -final as ( - - select - stg_nodes.unique_id as node_unique_id, - stg_nodes.name as node_name, - stg_nodes.resource_type as node_resource_type, - stg_nodes.file_path as node_file_path, - stg_nodes.is_enabled as node_is_enabled, - stg_nodes.database as node_database, - stg_nodes.schema as node_schema, - stg_nodes.package_name as node_package_name, - stg_nodes.alias as node_alias, - stg_columns.name as name, - stg_columns.description as description, - stg_columns.data_type as data_type, - stg_columns.quote as quote - - from stg_columns - - inner join stg_nodes - on stg_nodes.unique_id = stg_columns.node_unique_id - -) - -select * from final From 438b19fa867bb5c564bda31e821d0c8ee91ae352 Mon Sep 17 00:00:00 2001 From: Grace Goheen <53586774+graciegoheen@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:47:44 -0700 Subject: [PATCH 14/15] Apply suggestions from code review --- macros/unpack/get_column_values.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/unpack/get_column_values.sql b/macros/unpack/get_column_values.sql index e7aa4ed2..f4d71c75 100644 --- a/macros/unpack/get_column_values.sql +++ b/macros/unpack/get_column_values.sql @@ -23,8 +23,8 @@ wrap_string_with_quotes(node.unique_id), wrap_string_with_quotes(dbt.escape_single_quotes(column.name)), wrap_string_with_quotes(dbt.escape_single_quotes(column.description)), - 'null' if not column.data_type else wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), - 'null' if not column.quote else wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) + wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), + wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) ] %} From e8d03e780fa9dc2ffb423b14bca586e0f743cab6 Mon Sep 17 00:00:00 2001 From: Grace Goheen <53586774+graciegoheen@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:48:19 -0700 Subject: [PATCH 15/15] Update macros/unpack/get_column_values.sql --- macros/unpack/get_column_values.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/unpack/get_column_values.sql b/macros/unpack/get_column_values.sql index f4d71c75..d1607b57 100644 --- a/macros/unpack/get_column_values.sql +++ b/macros/unpack/get_column_values.sql @@ -24,7 +24,7 @@ wrap_string_with_quotes(dbt.escape_single_quotes(column.name)), wrap_string_with_quotes(dbt.escape_single_quotes(column.description)), wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), - wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) + wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) ] %}