From a4a17325ffec90ad3a468f76918941d9908cb151 Mon Sep 17 00:00:00 2001
From: Mike Alfare <13974384+mikealfare@users.noreply.github.com>
Date: Tue, 7 Nov 2023 20:47:20 -0500
Subject: [PATCH] ADAP-1017: Fix configuration change monitoring for scenarios
with no changes (#1009)
* changie
* add a test demonstrating the issue
* check that the change collection has changes instead of checking that it exists
* remove the partition config value from the config change since it is not hashable
(cherry picked from commit 64e042a2349f17342f7af96d559e4e3a5138a49e)
---
.../unreleased/Fixes-20231107-174352.yaml | 7 ++++++
dbt/adapters/bigquery/relation.py | 5 ++--
.../bigquery/relation_configs/_partition.py | 2 +-
.../adapter/materialized_view_tests/_files.py | 11 +++++++++
.../test_materialized_view.py | 24 +++++++++++++++++++
5 files changed, 46 insertions(+), 3 deletions(-)
create mode 100644 .changes/unreleased/Fixes-20231107-174352.yaml
diff --git a/.changes/unreleased/Fixes-20231107-174352.yaml b/.changes/unreleased/Fixes-20231107-174352.yaml
new file mode 100644
index 000000000..80592758d
--- /dev/null
+++ b/.changes/unreleased/Fixes-20231107-174352.yaml
@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fixed issue where materialized views were failing on re-run with minimal config
+ parameters
+time: 2023-11-07T17:43:52.972135-05:00
+custom:
+ Author: "mikealfare"
+ Issue: "1007"
diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py
index a689e76fc..c14dba238 100644
--- a/dbt/adapters/bigquery/relation.py
+++ b/dbt/adapters/bigquery/relation.py
@@ -84,9 +84,10 @@ def materialized_view_config_changeset(
)
if new_materialized_view.partition != existing_materialized_view.partition:
+ # the existing PartitionConfig is not hashable, but since we need to do
+ # a full refresh either way, we don't need to provide a context
config_change_collection.partition = BigQueryPartitionConfigChange(
action=RelationConfigChangeAction.alter,
- context=new_materialized_view.partition,
)
if new_materialized_view.cluster != existing_materialized_view.cluster:
@@ -95,7 +96,7 @@ def materialized_view_config_changeset(
context=new_materialized_view.cluster,
)
- if config_change_collection:
+ if config_change_collection.has_changes:
return config_change_collection
return None
diff --git a/dbt/adapters/bigquery/relation_configs/_partition.py b/dbt/adapters/bigquery/relation_configs/_partition.py
index 094e4f1c4..0d0ee23a1 100644
--- a/dbt/adapters/bigquery/relation_configs/_partition.py
+++ b/dbt/adapters/bigquery/relation_configs/_partition.py
@@ -152,7 +152,7 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]:
@dataclass(frozen=True, eq=True, unsafe_hash=True)
class BigQueryPartitionConfigChange(RelationConfigChange):
- context: Optional[PartitionConfig]
+ context: Optional[Any] = None
@property
def requires_full_refresh(self) -> bool:
diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py
index 1ee64269d..86714036a 100644
--- a/tests/functional/adapter/materialized_view_tests/_files.py
+++ b/tests/functional/adapter/materialized_view_tests/_files.py
@@ -67,3 +67,14 @@
record_valid_date
from {{ ref('my_seed') }}
"""
+
+
+MY_MINIMAL_MATERIALIZED_VIEW = """
+{{
+ config(
+ materialized = 'materialized_view',
+ )
+}}
+
+select * from {{ ref('my_seed') }}
+"""
diff --git a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py
index 7ca90983e..4e980c2e4 100644
--- a/tests/functional/adapter/materialized_view_tests/test_materialized_view.py
+++ b/tests/functional/adapter/materialized_view_tests/test_materialized_view.py
@@ -4,6 +4,7 @@
from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic
from tests.functional.adapter.materialized_view_tests._mixin import BigQueryMaterializedViewMixin
+from tests.functional.adapter.materialized_view_tests import _files
class TestBigqueryMaterializedViewsBasic(BigQueryMaterializedViewMixin, MaterializedViewBasic):
@@ -27,3 +28,26 @@ def test_materialized_view_only_updates_after_refresh(
self, project, my_materialized_view, my_seed
):
pass
+
+
+class TestMaterializedViewRerun:
+ """
+ This addresses: https://github.com/dbt-labs/dbt-bigquery/issues/1007
+
+ This effectively tests that defaults get properly set so that the run is idempotent.
+ If the defaults are not properly set, changes could appear when there are no changes
+ and cause unexpected scenarios.
+ """
+
+ @pytest.fixture(scope="class", autouse=True)
+ def models(self):
+ return {"my_minimal_materialized_view.sql": _files.MY_MINIMAL_MATERIALIZED_VIEW}
+
+ @pytest.fixture(scope="class", autouse=True)
+ def seeds(self):
+ return {"my_seed.csv": _files.MY_SEED}
+
+ def test_minimal_materialized_view_is_idempotent(self, project):
+ run_dbt(["seed"])
+ run_dbt(["run"])
+ run_dbt(["run"])