Skip to content

Commit

Permalink
Added SQL Server specific uploads
Browse files Browse the repository at this point in the history
  • Loading branch information
bvkabn committed Sep 18, 2024
1 parent a85d3e8 commit 7979318
Show file tree
Hide file tree
Showing 11 changed files with 546 additions and 87 deletions.
50 changes: 43 additions & 7 deletions macros/upload_individual_datasets/upload_exposures.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{% macro upload_exposures(exposures) -%}
{{ return(adapter.dispatch('get_exposures_dml_sql', 'dbt_artifacts')(exposures)) }}
{{ return(adapter.dispatch("get_exposures_dml_sql", "dbt_artifacts")(exposures)) }}
{%- endmacro %}

{% macro default__get_exposures_dml_sql(exposures) -%}
Expand Down Expand Up @@ -47,8 +47,7 @@
{%- endfor %}
{% endset %}
{{ exposure_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

Expand Down Expand Up @@ -80,8 +79,7 @@
{%- endfor %}
{% endset %}
{{ exposure_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{%- endmacro %}

Expand Down Expand Up @@ -114,7 +112,45 @@
{%- endfor %}
{% endset %}
{{ exposure_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{%- endmacro %}

{% macro sqlserver__get_exposures_dml_sql(exposures) -%}

{% if exposures != [] %}
{% set exposure_values %}
select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"
from ( values
{% for exposure in exposures -%}
(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ exposure.unique_id | replace("'","''") }}', {# node_id #}
'{{ run_started_at }}', {# run_started_at #}
'{{ exposure.name | replace("'","''") }}', {# name #}
'{{ exposure.type }}', {# type #}
'{{ tojson(exposure.owner) }}', {# owner #}
'{{ exposure.maturity }}', {# maturity #}
'{{ exposure.original_file_path }}', {# path #}
'{{ exposure.description | replace("'","''") }}', {# description #}
'{{ exposure.url }}', {# url #}
'{{ exposure.package_name }}', {# package_name #}
'{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #}
'{{ tojson(exposure.tags) }}', {# tags #}
{% if var('dbt_artifacts_exclude_all_results', false) %}
null
{% else %}
'{{ tojson(exposure) | replace("'", "''") }}' {# all_results #}
{% endif %}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}

) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14")

{% endset %}
{{ exposure_values }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

91 changes: 83 additions & 8 deletions macros/upload_individual_datasets/upload_invocations.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,22 @@

{# Need to remove keys with results that can't be handled properly #}
{# warn_error_options - returns a python object in 1.5 #}
{% if 'warn_error_options' in invocation_args_dict %}
{% if "warn_error_options" in invocation_args_dict %}
{% if invocation_args_dict.warn_error_options is not string %}
{% if invocation_args_dict.warn_error_options.include %}
{% set include_options = invocation_args_dict.warn_error_options.include %}
{% else %}
{% set include_options = '' %}
{% else %} {% set include_options = "" %}
{% endif %}
{% if invocation_args_dict.warn_error_options.exclude %}
{% set exclude_options = invocation_args_dict.warn_error_options.exclude %}
{% else %}
{% set exclude_options = '' %}
{% else %} {% set exclude_options = "" %}
{% endif %}
{% set warn_error_options = {'include': include_options, 'exclude': exclude_options} %}
{%- do invocation_args_dict.update({'warn_error_options': warn_error_options}) %}
{% set warn_error_options = {"include": include_options, "exclude": exclude_options} %}
{%- do invocation_args_dict.update({"warn_error_options": warn_error_options}) %}
{% endif %}
{% endif %}

{{ return(adapter.dispatch('get_invocations_dml_sql', 'dbt_artifacts')()) }}
{{ return(adapter.dispatch("get_invocations_dml_sql", "dbt_artifacts")()) }}
{%- endmacro %}

{% macro default__get_invocations_dml_sql() -%}
Expand Down Expand Up @@ -221,3 +219,80 @@
{{ invocation_values }}

{% endmacro -%}


{% macro sqlserver__get_invocations_dml_sql() -%}
{% set invocation_values %}
select
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
nullif("11", ''),
nullif("12", ''),
nullif("13", ''),
nullif("14", ''),
nullif("15", ''),
"16",
"17",
"18",
"19"
from (values
(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ dbt_version }}', {# dbt_version #}
'{{ project_name }}', {# project_name #}
'{{ run_started_at }}', {# run_started_at #}
'{{ flags.WHICH }}', {# dbt_command #}
'{{ flags.FULL_REFRESH }}', {# full_refresh_flag #}
'{{ target.profile_name }}', {# target_profile_name #}
'{{ target.name }}', {# target_name #}
'{{ target.schema }}', {# target_schema #}
{{ target.threads }}, {# target_threads #}

'{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #}
'{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #}
'{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #}
'{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #}
'{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","''") }}', {# dbt_cloud_run_reason #}
{% if var('env_vars', none) %}
{% set env_vars_dict = {} %}
{% for env_variable in var('env_vars') %}
{% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %}
{% endfor %}
'{{ tojson(env_vars_dict) }}', {# env_vars #}
{% else %}
null, {# env_vars #}
{% endif %}
{% if var('dbt_vars', none) %}
{% set dbt_vars_dict = {} %}
{% for dbt_var in var('dbt_vars') %}
{% do dbt_vars_dict.update({dbt_var: (var(dbt_var, '') | replace("'", "''"))}) %}
{% endfor %}
'{{ tojson(dbt_vars_dict) }}', {# dbt_vars #}
{% else %}
null, {# dbt_vars #}
{% endif %}
'{{ tojson(invocation_args_dict) | replace("'", "''") }}', {# invocation_args #}

{% set metadata_env = {} %}
{% for key, value in dbt_metadata_envs.items() %}
{% do metadata_env.update({key: (value | replace("'", "''"))}) %}
{% endfor %}
'{{ tojson(metadata_env) }}' {# dbt_custom_envs #}

)

) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19")

{% endset %}
{{ invocation_values }}

{% endmacro -%}

61 changes: 52 additions & 9 deletions macros/upload_individual_datasets/upload_model_executions.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{% macro upload_model_executions(models) -%}
{{ return(adapter.dispatch('get_model_executions_dml_sql', 'dbt_artifacts')(models)) }}
{{ return(adapter.dispatch("get_model_executions_dml_sql", "dbt_artifacts")(models)) }}
{%- endmacro %}

{% macro default__get_model_executions_dml_sql(models) -%}
Expand Down Expand Up @@ -57,8 +57,7 @@
{%- endfor %}
{% endset %}
{{ model_execution_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

Expand Down Expand Up @@ -99,8 +98,7 @@
{%- endfor %}
{% endset %}
{{ model_execution_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{%- endmacro %}

Expand Down Expand Up @@ -158,8 +156,7 @@
{%- endfor %}
{% endset %}
{{ model_execution_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

Expand Down Expand Up @@ -199,7 +196,53 @@
{%- endfor %}
{% endset %}
{{ model_execution_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{%- endmacro %}


{% macro sqlserver__get_model_executions_dml_sql(models) -%}
{% if models != [] %}
{% set model_execution_values %}
select
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16"
from ( values
{% for model in models -%}
(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ model.node.unique_id }}', {# node_id #}
'{{ run_started_at }}', {# run_started_at #}

{% set config_full_refresh = model.node.config.full_refresh %}
{% if config_full_refresh is none %}
{% set config_full_refresh = flags.FULL_REFRESH %}
{% endif %}
'{{ config_full_refresh }}', {# was_full_refresh #}

'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
null, -- rows_affected not available {# Only available in Snowflake & BigQuery #}
'{{ model.node.config.materialized }}', {# materialization #}
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message | replace("'", "''") }}', {# message #}
'{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
) v ( "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" )

{% endset %}
{{ model_execution_values }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

55 changes: 48 additions & 7 deletions macros/upload_individual_datasets/upload_models.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{% macro upload_models(models) -%}
{{ return(adapter.dispatch('get_models_dml_sql', 'dbt_artifacts')(models)) }}
{{ return(adapter.dispatch("get_models_dml_sql", "dbt_artifacts")(models)) }}
{%- endmacro %}

{% macro default__get_models_dml_sql(models) -%}
Expand Down Expand Up @@ -51,8 +51,7 @@
{%- endfor %}
{% endset %}
{{ model_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

Expand Down Expand Up @@ -87,8 +86,7 @@
{%- endfor %}
{% endset %}
{{ model_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{%- endmacro %}

Expand Down Expand Up @@ -123,7 +121,50 @@
{%- endfor %}
{% endset %}
{{ model_values }}
{% else %}
{{ return("") }}
{% else %} {{ return("") }}
{% endif %}
{%- endmacro %}


{% macro sqlserver__get_models_dml_sql(models) -%}

{% if models != [] %}
{% set model_values %}
select
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"
from ( values
{% for model in models -%}
{% set model_copy = model.copy() -%}
{% do model_copy.pop('raw_code', None) %}
(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ model_copy.unique_id }}', {# node_id #}
'{{ run_started_at }}', {# run_started_at #}
'{{ model_copy.database }}', {# database #}
'{{ model_copy.schema }}', {# schema #}
'{{ model_copy.name }}', {# name #}
'{{ tojson(model_copy.depends_on.nodes) }}', {# depends_on_nodes #}
'{{ model_copy.package_name }}', {# package_name #}
'{{ model_copy.original_file_path }}', {# path #}
'{{ model_copy.checksum.checksum }}', {# checksum #}
'{{ model_copy.config.materialized }}', {# materialization #}
'{{ tojson(model_copy.tags) }}', {# tags #}
'{{ tojson(model_copy.config.meta) | replace("'","''") }}', {# meta #}
'{{ model_copy.alias }}', {# alias #}
{% if var('dbt_artifacts_exclude_all_results', false) %}
null
{% else %}
'{{ tojson(model_copy) | replace("'","''") }}' {# all_results #}
{% endif %}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}

) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15")

{% endset %}
{{ model_values }}
{% else %} {{ return("") }}
{% endif %}
{% endmacro -%}

Loading

0 comments on commit 7979318

Please sign in to comment.