diff --git a/.changes/unreleased/Features-20231108-140752.yaml b/.changes/unreleased/Features-20231108-140752.yaml new file mode 100644 index 000000000..2a9a41a07 --- /dev/null +++ b/.changes/unreleased/Features-20231108-140752.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support allow_non_incremental_definition option in BigQuery materialized views +time: 2023-11-08T14:07:52.28972-05:00 +custom: + Author: bnaul + Issue: "672" diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 36222f44f..ad8a6a13e 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -101,6 +101,7 @@ class BigqueryConfig(AdapterConfig): max_staleness: Optional[str] = None enable_list_inference: Optional[bool] = None intermediate_format: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = None class BigQueryAdapter(BaseAdapter): diff --git a/dbt/adapters/bigquery/relation.py b/dbt/adapters/bigquery/relation.py index 086b4a2aa..99aa2036c 100644 --- a/dbt/adapters/bigquery/relation.py +++ b/dbt/adapters/bigquery/relation.py @@ -90,8 +90,16 @@ def materialized_view_config_changeset( new_materialized_view = cls.materialized_view_from_relation_config(relation_config) if new_materialized_view.options != existing_materialized_view.options: + # allow_non_incremental_definition cannot be changed via ALTER, must recreate + if ( + new_materialized_view.options.allow_non_incremental_definition + == existing_materialized_view.options.allow_non_incremental_definition + ): + action = RelationConfigChangeAction.alter + else: + action = RelationConfigChangeAction.create config_change_collection.options = BigQueryOptionsConfigChange( - action=RelationConfigChangeAction.alter, + action=action, context=new_materialized_view.options, ) diff --git a/dbt/adapters/bigquery/relation_configs/_options.py b/dbt/adapters/bigquery/relation_configs/_options.py index f0272df08..2f51acedf 100644 --- a/dbt/adapters/bigquery/relation_configs/_options.py +++ b/dbt/adapters/bigquery/relation_configs/_options.py @@ -9,6 +9,7 @@ from dbt.adapters.bigquery.relation_configs._base import BigQueryBaseRelationConfig from dbt.adapters.bigquery.utility import bool_setting, float_setting, sql_escape +from dbt.adapters.relation_configs import RelationConfigChangeAction @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -22,6 +23,7 @@ class BigQueryOptionsConfig(BigQueryBaseRelationConfig): refresh_interval_minutes: Optional[float] = 30 expiration_timestamp: Optional[datetime] = None max_staleness: Optional[str] = None + allow_non_incremental_definition: Optional[bool] = False kms_key_name: Optional[str] = None description: Optional[str] = None labels: Optional[Dict[str, str]] = None @@ -58,6 +60,7 @@ def array(x): "refresh_interval_minutes": numeric, "expiration_timestamp": interval, "max_staleness": interval, + "allow_non_incremental_definition": boolean, "kms_key_name": string, "description": escaped_string, "labels": array, @@ -85,6 +88,7 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> Self: "refresh_interval_minutes": float_setting, "expiration_timestamp": None, "max_staleness": None, + "allow_non_incremental_definition": bool_setting, "kms_key_name": None, "description": None, "labels": None, @@ -115,6 +119,7 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> Dict[str, Any "refresh_interval_minutes", "expiration_timestamp", "max_staleness", + "allow_non_incremental_definition", "kms_key_name", "description", "labels", @@ -139,7 +144,12 @@ def parse_bq_table(cls, table: BigQueryTable) -> Dict[str, Any]: "enable_refresh": table.mview_enable_refresh, "refresh_interval_minutes": table.mview_refresh_interval.seconds / 60, "expiration_timestamp": table.expires, - "max_staleness": None, + "max_staleness": ( + f"INTERVAL '{table._properties.get('maxStaleness')}' YEAR TO SECOND" + if table._properties.get('maxStaleness') + else None + ), + "allow_non_incremental_definition": table._properties.get("materializedView", {}).get("allowNonIncrementalDefinition"), "description": table.description, } @@ -158,4 +168,4 @@ class BigQueryOptionsConfigChange(RelationConfigChange): @property def requires_full_refresh(self) -> bool: - return False + return self.action != RelationConfigChangeAction.alter diff --git a/tests/functional/adapter/materialized_view_tests/_files.py b/tests/functional/adapter/materialized_view_tests/_files.py index 86714036a..9f8fe14d3 100644 --- a/tests/functional/adapter/materialized_view_tests/_files.py +++ b/tests/functional/adapter/materialized_view_tests/_files.py @@ -36,7 +36,8 @@ cluster_by=["id", "value"], enable_refresh=True, refresh_interval_minutes=60, - max_staleness="INTERVAL 45 MINUTE" + max_staleness="INTERVAL 45 MINUTE", + allow_non_incremental_definition=True ) }} select id,