Skip to content

Commit

Permalink
Add functional tests for custom incremental strategies names 'microba…
Browse files Browse the repository at this point in the history
…tch' (#10716)

* Add functional tests for custom incremental strategies names 'microbatch'

* Point dev-requirement of `dbt-adapters` back to the main branch

The associated branch/PR in `dbt-adapters` that we were previously
pointing to has been merged. Thus we can point back to `main` again.

---------

Co-authored-by: Quigley Malcolm <[email protected]>
  • Loading branch information
MichelleArk and QMalcolm authored Sep 16, 2024
1 parent 1e20772 commit 77aeb3e
Showing 1 changed file with 76 additions and 0 deletions.
76 changes: 76 additions & 0 deletions tests/functional/microbatch/test_microbatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
patch_microbatch_end_time,
relation_from_name,
run_dbt,
run_dbt_and_capture,
write_file,
)

Expand Down Expand Up @@ -69,6 +70,81 @@
select * from {{ source('seed_sources', 'raw_source') }}
"""

custom_microbatch_strategy = """
{% macro get_incremental_microbatch_sql(arg_dict) %}
{% do log('custom microbatch strategy', info=True) %}
{%- set dest_cols_csv = get_quoted_csv(arg_dict["dest_columns"] | map(attribute="name")) -%}
insert into {{ arg_dict["target_relation"] }} ({{ dest_cols_csv }})
(
select {{ dest_cols_csv }}
from {{ arg_dict["temp_relation"] }}
)
{% endmacro %}
"""


class BaseMicrobatchCustomUserStrategy:
@pytest.fixture(scope="class")
def models(self):
return {
"input_model.sql": input_model_sql,
"microbatch_model.sql": microbatch_model_sql,
}

@pytest.fixture(scope="class")
def macros(self):
return {"microbatch.sql": custom_microbatch_strategy}


class TestMicrobatchCustomUserStrategyDefault(BaseMicrobatchCustomUserStrategy):
def test_use_custom_microbatch_strategy_by_default(self, project):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: []
):
# Initial run
run_dbt(["run"])

# Incremental run uses custom strategy
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs


class TestMicrobatchCustomUserStrategyEnvVarTrueValid(BaseMicrobatchCustomUserStrategy):
@mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"})
def test_use_custom_microbatch_strategy_env_var_true_invalid_incremental_strategy(
self, project
):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: ["microbatch"]
):
# Initial run
run_dbt(["run"])

# Incremental run uses custom strategy
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs


# TODO: Consider a behaviour flag here if DBT_EXPERIMENTAL_MICROBATCH is removed
# Since this causes an exception prior to using an override
class TestMicrobatchCustomUserStrategyEnvVarTrueInvalid(BaseMicrobatchCustomUserStrategy):
@mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_MICROBATCH": "True"})
def test_use_custom_microbatch_strategy_env_var_true_invalid_incremental_strategy(
self, project
):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: []
):
# Initial run
run_dbt(["run"])

# Incremental run fails
_, logs = run_dbt_and_capture(["run"], expect_pass=False)
assert "'microbatch' is not valid" in logs


class TestMicrobatchCLI:
@pytest.fixture(scope="class")
Expand Down

0 comments on commit 77aeb3e

Please sign in to comment.