Skip to content

Commit

Permalink
Delimited schema and database because these are reserved keywords
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvkabn
bvkabn committed Sep 18, 2024
1 parent 7979318 commit a24997d
Show file tree
Hide file tree
Showing 25 changed files with 302 additions and 338 deletions.
63 changes: 22 additions & 41 deletions macros/migration/migrate_from_v0_to_v1.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
node_id,
query_completed_at,
rows_affected,
schema,
"schema",
status,
thread_id,
total_node_runtime,
Expand All @@ -34,9 +34,7 @@
{% endset %}

{{ log("Migrating model_executions", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_model_executions }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_model_executions }} {%- endcall -%}

{% set migrate_tests %}
insert into {{new_database}}.{{new_schema}}.tests (
Expand All @@ -62,9 +60,7 @@
{% endset %}

{{ log("Migrating tests", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_tests }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_tests }} {%- endcall -%}

{% set migrate_test_executions %}
insert into {{new_database}}.{{new_schema}}.test_executions (
Expand Down Expand Up @@ -96,22 +92,20 @@
{% endset %}

{{ log("Migrating test_executions", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_test_executions }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_test_executions }} {%- endcall -%}

{% set migrate_models %}
insert into {{new_database}}.{{new_schema}}.models (
checksum,
command_invocation_id,
database,
"database",
depends_on_nodes,
materialization,
name,
node_id,
package_name,
path,
schema,
"schema",
run_started_at
)
select
Expand All @@ -130,20 +124,18 @@
{% endset %}

{{ log("Migrating models", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_models }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_models }} {%- endcall -%}

{% set migrate_seeds %}
insert into {{new_database}}.{{new_schema}}.seeds (
checksum,
command_invocation_id,
database,
"database",
name,
node_id,
package_name,
path,
schema,
"schema",
run_started_at
)
select
Expand All @@ -160,9 +152,7 @@
{% endset %}

{{ log("Migrating seeds", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_seeds }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_seeds }} {%- endcall -%}

{% set migrate_seed_executions %}
insert into {{new_database}}.{{new_schema}}.seed_executions (
Expand All @@ -173,7 +163,7 @@
node_id,
query_completed_at,
rows_affected,
schema,
"schema",
status,
thread_id,
total_node_runtime,
Expand All @@ -198,9 +188,7 @@
{% endset %}

{{ log("Migrating seed_executions", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_seed_executions }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_seed_executions }} {%- endcall -%}

{% set migrate_exposures %}
insert into {{new_database}}.{{new_schema}}.exposures (
Expand Down Expand Up @@ -235,21 +223,19 @@
{% endset %}

{{ log("Migrating exposures", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_exposures }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_exposures }} {%- endcall -%}

{% set migrate_snapshots %}
insert into {{new_database}}.{{new_schema}}.snapshots (
checksum,
command_invocation_id,
database,
"database",
depends_on_nodes,
name,
node_id,
package_name,
path,
schema,
"schema",
strategy,
run_started_at
)
Expand All @@ -269,9 +255,7 @@
{% endset %}

{{ log("Migrating snapshots", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_snapshots }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_snapshots }} {%- endcall -%}

{% set migrate_snapshot_executions %}
insert into {{new_database}}.{{new_schema}}.snapshot_executions (
Expand All @@ -282,7 +266,7 @@
node_id,
query_completed_at,
rows_affected,
schema,
"schema",
status,
thread_id,
total_node_runtime,
Expand All @@ -307,21 +291,19 @@
{% endset %}

{{ log("Migrating snapshot_executions", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_snapshot_executions }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_snapshot_executions }} {%- endcall -%}

{% set migrate_sources %}
insert into {{new_database}}.{{new_schema}}.sources (
command_invocation_id,
database,
"database",
freshness,
identifier,
loaded_at_field,
loader,
name,
node_id,
schema,
"schema",
source_name,
run_started_at
)
Expand All @@ -341,9 +323,8 @@
{% endset %}

{{ log("Migrating sources", info=True) }}
{%- call statement(auto_begin=True) -%}
{{ migrate_sources }}
{%- endcall -%}
{%- call statement(auto_begin=True) -%} {{ migrate_sources }} {%- endcall -%}

{{ log("Migration complete. You can now safely delete any data from before 1.0.0", info=True) }}
{%- endmacro -%}

54 changes: 25 additions & 29 deletions macros/upload_results/get_column_name_lists.sql
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@

{#
These are the column lists used as part of the upload macros - the order here should be the same
as the order in each individual `upload_dataset` macro.
#}

{% macro get_column_name_list(dataset) -%}

{% if dataset == 'exposures' %}
{% if dataset == "exposures" %}

(
command_invocation_id,
Expand All @@ -25,7 +23,7 @@
all_results
)

{% elif dataset == 'invocations' %}
{% elif dataset == "invocations" %}

(
command_invocation_id,
Expand All @@ -49,7 +47,7 @@
dbt_custom_envs
)

{% elif dataset == 'model_executions' %}
{% elif dataset == "model_executions" %}

(
command_invocation_id,
Expand All @@ -62,25 +60,24 @@
query_completed_at,
total_node_runtime,
rows_affected,
{% if target.type == 'bigquery' %}
bytes_processed,
{% if target.type == "bigquery" %} bytes_processed,
{% endif %}
materialization,
schema,
"schema",
name,
alias,
message,
adapter_response
)

{% elif dataset == 'models' %}
{% elif dataset == "models" %}

(
command_invocation_id,
node_id,
run_started_at,
database,
schema,
"database",
"schema",
name,
depends_on_nodes,
package_name,
Expand All @@ -93,8 +90,7 @@
all_results
)


{% elif dataset == 'seed_executions' %}
{% elif dataset == "seed_executions" %}

(
command_invocation_id,
Expand All @@ -108,21 +104,21 @@
total_node_runtime,
rows_affected,
materialization,
schema,
"schema",
name,
alias,
message,
adapter_response
)

{% elif dataset == 'seeds' %}
{% elif dataset == "seeds" %}

(
command_invocation_id,
node_id,
run_started_at,
database,
schema,
"database",
"schema",
name,
package_name,
path,
Expand All @@ -132,7 +128,7 @@
all_results
)

{% elif dataset == 'snapshot_executions' %}
{% elif dataset == "snapshot_executions" %}

(
command_invocation_id,
Expand All @@ -146,21 +142,21 @@
total_node_runtime,
rows_affected,
materialization,
schema,
"schema",
name,
alias,
message,
adapter_response
)

{% elif dataset == 'snapshots' %}
{% elif dataset == "snapshots" %}

(
command_invocation_id,
node_id,
run_started_at,
database,
schema,
"database",
"schema",
name,
depends_on_nodes,
package_name,
Expand All @@ -172,14 +168,14 @@
all_results
)

{% elif dataset == 'sources' %}
{% elif dataset == "sources" %}

(
command_invocation_id,
node_id,
run_started_at,
database,
schema,
"database",
"schema",
source_name,
loader,
name,
Expand All @@ -189,7 +185,7 @@
all_results
)

{% elif dataset == 'test_executions' %}
{% elif dataset == "test_executions" %}

(
command_invocation_id,
Expand All @@ -207,7 +203,7 @@
adapter_response
)

{% elif dataset == 'tests' %}
{% elif dataset == "tests" %}

(
command_invocation_id,
Expand All @@ -223,8 +219,8 @@

{% else %}

/* No column list available */

/* No column list available */
{% endif %}

{%- endmacro %}

12 changes: 12 additions & 0 deletions macros/upload_results/insert_into_metadata_table.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,17 @@

{%- endmacro %}

{% macro sqlserver__insert_into_metadata_table(relation, fields, content) -%}

{% set insert_into_table_query %}
insert into {{ relation }} {{ fields }}
{{ content }}
{% endset %}

{% do run_query(insert_into_table_query) %}

{%- endmacro %}

{% macro default__insert_into_metadata_table(relation, fields, content) -%}
{%- endmacro %}

Loading

0 comments on commit a24997d

Please sign in to comment.