Release 0.11.0

Datavault-UK · Jun 7, 2024 · 6a929dc · 6a929dc
2 parents f3d29cd + de32631
commit 6a929dc
Show file tree

Hide file tree

Showing 119 changed files with 727 additions and 1,005 deletions.
diff --git a/NOTICE b/NOTICE
@@ -1,5 +1,5 @@
 AutomateDV (f.k.a dbtvault)
-Copyright 2019-2023 Business Thinking Ltd (Trading as Datavault).
+Copyright 2019-2024 Business Thinking Ltd (Trading as Datavault).
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,14 +1,14 @@
 name: automate_dv
-version: 0.10.2
-require-dbt-version: [">=1.0.0", "<2.0.0"]
+version: 0.11.0
+require-dbt-version: [ ">=1.0.0", "<2.0.0" ]
 config-version: 2
 
-model-paths: ["models"]
-analysis-paths: ["analyses"]
-test-paths: ["tests"]
-seed-paths: ["seeds"]
-macro-paths: ["macros"]
-docs-paths: ["docs"]
+model-paths: [ "models" ]
+analysis-paths: [ "analyses" ]
+test-paths: [ "tests" ]
+seed-paths: [ "seeds" ]
+macro-paths: [ "macros" ]
+docs-paths: [ "docs" ]
 
 target-path: "target"
 clean-targets:

diff --git a/macros/internal/helpers/dateadd.sql b/macros/internal/helpers/dateadd.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/is_checks.sql b/macros/internal/helpers/is_checks.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/materialisations/error_messages.sql → ...ternal/helpers/logging/error_messages.sql b/macros/materialisations/error_messages.sql → ...ternal/helpers/logging/error_messages.sql
@@ -1,31 +1,9 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
 
-{% macro wrap_warning(warning_message) %}
-
-    {%- set new_message = [] -%}
-    {%- set length_list = [] -%}
-
-    {%- for ln in warning_message.split('\n') -%}
-        {%- do new_message.append((ln | trim)) -%}
-        {%- do length_list.append((ln | length)) -%}
-    {%- endfor -%}
-
-    {%- set max_line_length = length_list | max -%}
-    {%- set padding_length = (max_line_length - 7) // 2 -%}
-
-    {%- set border = modules.itertools.repeat('=', padding_length) | join ('') ~ 'WARNING' ~ modules.itertools.repeat('=', padding_length) | join ('') -%}
-
-    {%- set wrapped_message = '\n' ~ border ~ '\n' ~ new_message | join('\n') ~ '\n' ~ border -%}
-
-    {%- do return(wrapped_message) -%}
-
-{% endmacro %}
-
-
 {%- macro datepart_too_small_error(period) -%}
 
     {%- set message -%}
@@ -35,10 +13,7 @@
     Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.raise_compiler_error(message) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_error(message) -%}
 {%- endmacro -%}
 
 
@@ -51,10 +26,7 @@
     Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.warn(automate_dv.wrap_warning(message)) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}
 
 
@@ -67,10 +39,7 @@
     Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/
     {%- endset %}
 
-    {%- if execute -%}
-    {{- exceptions.raise_compiler_error(message) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}
 
 
@@ -82,10 +51,7 @@
     Using this functionality in a live production setting may result in unpredictable outcomes, data loss, or system instability.
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.warn(automate_dv.wrap_warning(message)) -}}
-    {%- endif -%}
-
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}
 
 
@@ -99,8 +65,32 @@
     We are actively working to get this fixed. Thank you for your understanding.
     {%- endset -%}
 
-    {%- if execute -%}
-    {{- exceptions.raise_compiler_error(automate_dv.wrap_warning(message)) -}}
-    {%- endif -%}
+    {%- do automate_dv.log_error(message) -%}
+{%- endmacro -%}
+
+
+{%- macro materialisation_deprecation_warning() -%}
+
+    {%- set message -%}
+        DEPRECATED: Since AutomateDV v0.11.0, vault_insert_by_x materialisations are now deprecated.
+            These materialisation were initially designed to provide an option for rapid iterative development of
+            incremental loading patterns in local environments for development and testing, allowing users to bypass
+            the need for a comprehensive PSA or delta-loading solution. They are being deprecated to encourage the use
+            of more robust solutions.
+    {%- endset -%}
+
+    {%- do automate_dv.log_warning(message) -%}
+{%- endmacro -%}
+
+
+{%- macro pit_bridge_deprecation_warning() -%}
+
+    {%- set message -%}
+        DEPRECATED: Since AutomateDV v0.11.0, the pit() and bridge() macros are now deprecated.
+            This is because they are not currently fit-for-purpose and need significant usability
+            and peformance improvements, as well as a design overhaul.
+            Improved implementations will be released in a future version of AutomateDV.
+    {%- endset -%}
 
+    {%- do automate_dv.log_warning(message) -%}
 {%- endmacro -%}
diff --git a/macros/internal/helpers/logging/log_relation_sources.sql b/macros/internal/helpers/logging/log_relation_sources.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
@@ -9,7 +9,7 @@
 
 {% macro default__log_relation_sources(relation, source_count) %}
 
-    {%- if 'docs' not in invocation_args_dict['rpc_method'] and execute -%}
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'docs' -%}
 
         {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}.{}".format(relation.database, relation.schema, relation.identifier),
                                                                         source_count)) -%}
@@ -18,7 +18,7 @@
 
 {% macro databricks__log_relation_sources(relation, source_count) %}
 
-    {%- if 'docs' not in invocation_args_dict['rpc_method'] and execute -%}
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'docs' -%}
 
         {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}".format(relation.schema, relation.identifier),
                                                                         source_count)) -%}

diff --git a/macros/internal/helpers/logging/logging_helpers.sql b/macros/internal/helpers/logging/logging_helpers.sql
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) Business Thinking Ltd. 2019-2024
+ * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
+ */
+
+
+{% macro wrap_message(message, type='WARNING') %}
+
+    {%- set new_message = [] -%}
+    {%- set length_list = [] -%}
+
+    {%- for ln in message.split('\n') -%}
+        {%- do new_message.append((ln | trim)) -%}
+        {%- do length_list.append((ln | length)) -%}
+    {%- endfor -%}
+
+    {%- set max_line_length = length_list | max -%}
+    {%- set padding_length = (max_line_length - 7) // 2 -%}
+
+    {%- set border = modules.itertools.repeat('=', padding_length) | join ('') ~ (type | upper) ~ modules.itertools.repeat('=', padding_length) | join ('') -%}
+
+    {%- set wrapped_message = '\n' ~ border ~ '\n' ~ new_message | join('\n') ~ '\n' ~ border -%}
+
+    {%- do return(wrapped_message) -%}
+
+{% endmacro %}
+
+
+{%- macro log_warning(message) -%}
+
+    {%- set message = automate_dv.wrap_message(message) -%}
+
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'compile' -%}
+        {%- do exceptions.warn(message) -%}
+    {%- endif -%}
+
+{%- endmacro -%}
+
+
+{%- macro log_error(message) -%}
+
+    {%- set message = automate_dv.wrap_message(message, type='ERROR') -%}
+
+    {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'compile' -%}
+        {%- do exceptions.raise_compiler_error(message) -%}
+    {%- endif -%}
+
+{%- endmacro -%}
diff --git a/macros/internal/helpers/prepend_generated_by.sql b/macros/internal/helpers/prepend_generated_by.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/stage_processing_macros/extract_column_names.sql b/macros/internal/helpers/stage_processing_macros/extract_column_names.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql b/macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/stage_processing_macros/print_list.sql b/macros/internal/helpers/stage_processing_macros/print_list.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql b/macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql b/macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql b/macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/helpers/timestamp_add.sql b/macros/internal/helpers/timestamp_add.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/alias.sql b/macros/internal/metadata_processing/alias.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/alias_all.sql b/macros/internal/metadata_processing/alias_all.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/as_constant.sql b/macros/internal/metadata_processing/as_constant.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/check_required_parameters.sql b/macros/internal/metadata_processing/check_required_parameters.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/concat_ws.sql b/macros/internal/metadata_processing/concat_ws.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 
@@ -11,6 +11,16 @@
 
 {%- macro default__concat_ws(string_list, separator="||") -%}
 
+CONCAT_WS('{{ separator }}',
+{%- for str_obj in string_list %}
+    {{ str_obj }}{% if not loop.last %},{% endif %}
+{%- endfor %}
+)
+
+{%- endmacro -%}
+
+{%- macro bigquery__concat_ws(string_list, separator="||") -%}
+
 CONCAT(
 {%- for str in string_list %}
     {{ str }}
@@ -20,13 +30,19 @@ CONCAT(
 
 {%- endmacro -%}
 
-{%- macro bigquery__concat_ws(string_list, separator="||") -%}
+{%- macro sqlserver__concat_ws(string_list, separator="||") -%}
 
     {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }}
 
 {%- endmacro -%}
 
-{%- macro sqlserver__concat_ws(string_list, separator="||") -%}
+ {%- macro postgres__concat_ws(string_list, separator="||") -%}
+
+    {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }}
+
+{%- endmacro -%}
+
+{%- macro databricks__concat_ws(string_list, separator="||") -%}
 
     {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }}
 

diff --git a/macros/internal/metadata_processing/escape_column_name.sql b/macros/internal/metadata_processing/escape_column_name.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/escape_column_names.sql b/macros/internal/metadata_processing/escape_column_names.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/expand_column_list.sql b/macros/internal/metadata_processing/expand_column_list.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/get_escape_characters.sql b/macros/internal/metadata_processing/get_escape_characters.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */
 

diff --git a/macros/internal/metadata_processing/multikey.sql b/macros/internal/metadata_processing/multikey.sql
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) Business Thinking Ltd. 2019-2023
+ * Copyright (c) Business Thinking Ltd. 2019-2024
  * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
  */