diff --git a/tests/functional/adapter/test_incremental_unique_id.py b/tests/functional/adapter/test_incremental_unique_id.py new file mode 100644 index 000000000..a485ba9a7 --- /dev/null +++ b/tests/functional/adapter/test_incremental_unique_id.py @@ -0,0 +1,6 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey + + +class TestUniqueKeyBigQuery(BaseIncrementalUniqueKey): + pass \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql deleted file mode 100644 index 7290b6c43..000000000 --- a/tests/integration/incremental_unique_id_test/models/duplicated_unary_unique_key_list.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key=['state', 'state'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql b/tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql deleted file mode 100644 index 5260e177c..000000000 --- a/tests/integration/incremental_unique_id_test/models/empty_str_unique_key.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='' - ) -}} - -select - * -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql deleted file mode 100644 index c582d532c..000000000 --- a/tests/integration/incremental_unique_id_test/models/empty_unique_key_list.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key=[] - ) -}} - -select * from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql b/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql deleted file mode 100644 index baa37c601..000000000 --- a/tests/integration/incremental_unique_id_test/models/expected/one_str__overwrite.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select - 'CT' as state, - 'Hartford' as county, - 'Hartford' as city, - cast('2022-02-14' as date) as last_visit_date -union all -select 'MA','Suffolk','Boston','2020-02-12' -union all -select 'NJ','Mercer','Trenton','2022-01-01' -union all -select 'NY','Kings','Brooklyn','2021-04-02' -union all -select 'NY','New York','Manhattan','2021-04-01' -union all -select 'PA','Philadelphia','Philadelphia','2021-05-21' \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql b/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql deleted file mode 100644 index baa37c601..000000000 --- a/tests/integration/incremental_unique_id_test/models/expected/unique_key_list__inplace_overwrite.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -select - 'CT' as state, - 'Hartford' as county, - 'Hartford' as city, - cast('2022-02-14' as date) as last_visit_date -union all -select 'MA','Suffolk','Boston','2020-02-12' -union all -select 'NJ','Mercer','Trenton','2022-01-01' -union all -select 'NY','Kings','Brooklyn','2021-04-02' -union all -select 'NY','New York','Manhattan','2021-04-01' -union all -select 'PA','Philadelphia','Philadelphia','2021-05-21' \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/no_unique_key.sql b/tests/integration/incremental_unique_id_test/models/no_unique_key.sql deleted file mode 100644 index 44a63e75c..000000000 --- a/tests/integration/incremental_unique_id_test/models/no_unique_key.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ - config( - materialized='incremental' - ) -}} - -select - * -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql deleted file mode 100644 index 52b4509f0..000000000 --- a/tests/integration/incremental_unique_id_test/models/nontyped_trinary_unique_key_list.sql +++ /dev/null @@ -1,19 +0,0 @@ --- for comparing against auto-typed seeds - -{{ - config( - materialized='incremental', - unique_key=['state', 'county', 'city'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql b/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql deleted file mode 100644 index d247aa341..000000000 --- a/tests/integration/incremental_unique_id_test/models/not_found_unique_key.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='thisisnotacolumn' - ) -}} - -select - * -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql deleted file mode 100644 index f1462a48f..000000000 --- a/tests/integration/incremental_unique_id_test/models/not_found_unique_key_list.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key=['state', 'thisisnotacolumn'] - ) -}} - -select * from {{ ref('seed') }} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql b/tests/integration/incremental_unique_id_test/models/str_unique_key.sql deleted file mode 100644 index 2f9fc2987..000000000 --- a/tests/integration/incremental_unique_id_test/models/str_unique_key.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key='state' - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql deleted file mode 100644 index 0359546bf..000000000 --- a/tests/integration/incremental_unique_id_test/models/trinary_unique_key_list.sql +++ /dev/null @@ -1,19 +0,0 @@ --- types needed to compare against expected model reliably - -{{ - config( - materialized='incremental', - unique_key=['state', 'county', 'city'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql b/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql deleted file mode 100644 index 7f5875f85..000000000 --- a/tests/integration/incremental_unique_id_test/models/unary_unique_key_list.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ - config( - materialized='incremental', - unique_key=['state'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql b/tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql deleted file mode 100644 index 6d515ca3a..000000000 --- a/tests/integration/incremental_unique_id_test/seeds/add_new_rows.sql +++ /dev/null @@ -1,9 +0,0 @@ --- insert two new rows, both of which should be in incremental model --- with any unique columns -insert into {schema}.seed - (state, county, city, last_visit_date) -values ('WA','King','Seattle','2022-02-01'); - -insert into {schema}.seed - (state, county, city, last_visit_date) -values ('CA','Los Angeles','Los Angeles','2022-02-01'); \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql b/tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql deleted file mode 100644 index bd73cee86..000000000 --- a/tests/integration/incremental_unique_id_test/seeds/duplicate_insert.sql +++ /dev/null @@ -1,5 +0,0 @@ --- insert new row, which should not be in incremental model --- with primary or first three columns unique -insert into {schema}.seed - (state, county, city, last_visit_date) -values ('CT','Hartford','Hartford','2022-02-14'); \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/seeds/seed.csv b/tests/integration/incremental_unique_id_test/seeds/seed.csv deleted file mode 100644 index b988827fb..000000000 --- a/tests/integration/incremental_unique_id_test/seeds/seed.csv +++ /dev/null @@ -1,7 +0,0 @@ -state,county,city,last_visit_date -CT,Hartford,Hartford,2020-09-23 -MA,Suffolk,Boston,2020-02-12 -NJ,Mercer,Trenton,2022-01-01 -NY,Kings,Brooklyn,2021-04-02 -NY,New York,Manhattan,2021-04-01 -PA,Philadelphia,Philadelphia,2021-05-21 \ No newline at end of file diff --git a/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py b/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py deleted file mode 100644 index a21d84327..000000000 --- a/tests/integration/incremental_unique_id_test/test_incremental_unique_id.py +++ /dev/null @@ -1,296 +0,0 @@ -from tests.integration.base import DBTIntegrationTest, use_profile -from dbt.contracts.results import RunStatus -from collections import namedtuple -from pathlib import Path - - -TestResults = namedtuple( - 'TestResults', - ['seed_count', 'model_count', 'seed_rows', 'inc_test_model_count', - 'opt_model_count', 'relation'], -) - - -class TestIncrementalUniqueKey(DBTIntegrationTest): - @property - def schema(self): - return 'incremental_unique_key' - - @property - def models(self): - return 'models' - - def update_incremental_model(self, incremental_model): - '''update incremental model after the seed table has been updated''' - model_result_set = self.run_dbt(['run', '--select', incremental_model]) - return len(model_result_set) - - def setup_test(self, seed, incremental_model, update_sql_file): - '''build a test case and return values for assertions''' - - # Idempotently create some number of seeds and incremental models - seed_count = len(self.run_dbt( - ['seed', '--select', seed, '--full-refresh'] - )) - model_count = len(self.run_dbt( - ['run', '--select', incremental_model, '--full-refresh'] - )) - - # Upate seed and return new row count - row_count_query = 'select * from {}.{}'.format( - self.unique_schema(), - seed - ) - self.run_sql_file(Path('seeds') / Path(update_sql_file + '.sql')) - seed_rows = len(self.run_sql(row_count_query, fetch='all')) - - inc_test_model_count = self.update_incremental_model( - incremental_model=incremental_model - ) - - return (seed_count, model_count, seed_rows, inc_test_model_count) - - def test_scenario_correctness(self, expected_fields, test_case_fields): - '''Invoke assertions to verify correct build functionality''' - # 1. test seed(s) should build afresh - self.assertEqual( - expected_fields.seed_count, test_case_fields.seed_count - ) - # 2. test model(s) should build afresh - self.assertEqual( - expected_fields.model_count, test_case_fields.model_count - ) - # 3. seeds should have intended row counts post update - self.assertEqual( - expected_fields.seed_rows, test_case_fields.seed_rows - ) - # 4. incremental test model(s) should be updated - self.assertEqual( - expected_fields.inc_test_model_count, - test_case_fields.inc_test_model_count - ) - # 5. extra incremental model(s) should be built; optional since - # comparison may be between an incremental model and seed - if (expected_fields.opt_model_count and - test_case_fields.opt_model_count): - self.assertEqual( - expected_fields.opt_model_count, - test_case_fields.opt_model_count - ) - # 6. result table should match intended result set (itself a relation) - self.assertTablesEqual( - expected_fields.relation, test_case_fields.relation - ) - - def stub_expected_fields( - self, relation, seed_rows, opt_model_count=None - ): - return TestResults( - seed_count=1, model_count=1, seed_rows=seed_rows, - inc_test_model_count=1, opt_model_count=opt_model_count, - relation=relation - ) - - def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): - '''should pass back error state when trying build an incremental - model whose unique key or keylist includes a column missing - from the incremental model''' - seed_count = len(self.run_dbt( - ['seed', '--select', 'seed', '--full-refresh'] - )) - # unique keys are not applied on first run, so two are needed - self.run_dbt( - ['run', '--select', incremental_model_name, '--full-refresh'], - expect_pass=True - ) - run_result = self.run_dbt( - ['run', '--select', incremental_model_name], - expect_pass=False - ).results[0] - - return run_result.status, run_result.message - - -class TestNoIncrementalUniqueKey(TestIncrementalUniqueKey): - @use_profile('bigquery') - def test__bigquery_no_unique_keys(self): - '''with no unique keys, seed and model should match''' - seed='seed' - seed_rows=8 - incremental_model='no_unique_key' - update_sql_file='add_new_rows' - - expected_fields = self.stub_expected_fields( - relation=seed, seed_rows=seed_rows - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=None, relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - -class TestIncrementalStrUniqueKey(TestIncrementalUniqueKey): - @use_profile('bigquery') - def test__bigquery_empty_str_unique_key(self): - '''with empty string for unique key, seed and model should match''' - seed='seed' - seed_rows=8 - incremental_model='empty_str_unique_key' - update_sql_file='add_new_rows' - - expected_fields = self.stub_expected_fields( - relation=seed, seed_rows=seed_rows - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=None, relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_one_unique_key(self): - '''with one unique key, model will overwrite existing row''' - seed='seed' - seed_rows=7 - incremental_model='str_unique_key' - update_sql_file='duplicate_insert' - expected_model='one_str__overwrite' - - expected_fields = self.stub_expected_fields( - relation=expected_model, seed_rows=seed_rows, opt_model_count=1 - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=self.update_incremental_model(expected_model), - relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_bad_unique_key(self): - '''expect compilation error from unique key not being a column''' - - err_msg = "Name thisisnotacolumn not found inside DBT_INTERNAL_SOURCE" - - (status, exc) = self.fail_to_build_inc_missing_unique_key_column( - incremental_model_name='not_found_unique_key' - ) - - self.assertEqual(status, RunStatus.Error) - self.assertTrue(err_msg in exc) - - -class TestIncrementalListUniqueKey(TestIncrementalUniqueKey): - @use_profile('bigquery') - def test__bigquery_empty_unique_key_list(self): - '''with no unique keys, seed and model should match''' - seed='seed' - seed_rows=8 - incremental_model='empty_unique_key_list' - update_sql_file='add_new_rows' - - expected_fields = self.stub_expected_fields( - relation=seed, seed_rows=seed_rows - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=None, relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_unary_unique_key_list(self): - '''with one unique key, model will overwrite existing row''' - seed='seed' - seed_rows=7 - incremental_model='unary_unique_key_list' - update_sql_file='duplicate_insert' - expected_model='unique_key_list__inplace_overwrite' - - expected_fields = self.stub_expected_fields( - relation=expected_model, seed_rows=seed_rows, opt_model_count=1 - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=self.update_incremental_model(expected_model), - relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_duplicated_unary_unique_key_list(self): - '''with two of the same unique key, model will overwrite existing row''' - seed='seed' - seed_rows=7 - incremental_model='duplicated_unary_unique_key_list' - update_sql_file='duplicate_insert' - expected_model='unique_key_list__inplace_overwrite' - - expected_fields = self.stub_expected_fields( - relation=expected_model, seed_rows=seed_rows, opt_model_count=1 - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=self.update_incremental_model(expected_model), - relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_trinary_unique_key_list(self): - '''with three unique keys, model will overwrite existing row''' - seed='seed' - seed_rows=7 - incremental_model='trinary_unique_key_list' - update_sql_file='duplicate_insert' - expected_model='unique_key_list__inplace_overwrite' - - expected_fields = self.stub_expected_fields( - relation=expected_model, seed_rows=seed_rows, opt_model_count=1 - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=self.update_incremental_model(expected_model), - relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_trinary_unique_key_list_no_update(self): - '''even with three unique keys, adding distinct rows to seed does not - cause seed and model to diverge''' - seed='seed' - seed_rows=8 - incremental_model='nontyped_trinary_unique_key_list' - update_sql_file='add_new_rows' - - expected_fields = self.stub_expected_fields( - relation=seed, seed_rows=seed_rows - ) - test_case_fields = TestResults( - *self.setup_test(seed, incremental_model, update_sql_file), - opt_model_count=None, relation=incremental_model - ) - - self.test_scenario_correctness(expected_fields, test_case_fields) - - @use_profile('bigquery') - def test__bigquery_bad_unique_key_list(self): - '''expect compilation error from unique key not being a column''' - - err_msg = "Name thisisnotacolumn not found inside DBT_INTERNAL_SOURCE" - - (status, exc) = self.fail_to_build_inc_missing_unique_key_column( - incremental_model_name='not_found_unique_key_list' - ) - - self.assertEqual(status, RunStatus.Error) - self.assertTrue(err_msg in exc) diff --git a/tox.ini b/tox.ini index 4b23d97e2..fad7d7e34 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ skip_install = true passenv = DBT_* BIGQUERY_TEST_* PYTEST_ADDOPTS commands = bigquery: {envpython} -m pytest {posargs} -m profile_bigquery tests/integration + bigquery: {envpython} -m pytest {posargs} tests/functional deps = -rdev_requirements.txt -e.