diff --git a/dbt/adapters/clickhouse/impl.py b/dbt/adapters/clickhouse/impl.py index 01e7183a..3b637926 100644 --- a/dbt/adapters/clickhouse/impl.py +++ b/dbt/adapters/clickhouse/impl.py @@ -10,8 +10,10 @@ from dbt.adapters.base.relation import BaseRelation, InformationSchema from dbt.adapters.sql import SQLAdapter from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType +from dbt.contracts.graph.nodes import ConstraintType from dbt.contracts.relation import RelationType +from dbt.events.functions import warn_or_error +from dbt.events.types import ConstraintNotSupported from dbt.exceptions import DbtInternalError, DbtRuntimeError, NotImplementedError from dbt.utils import executor, filter_null_values @@ -42,7 +44,7 @@ class ClickHouseAdapter(SQLAdapter): CONSTRAINT_SUPPORT = { ConstraintType.check: ConstraintSupport.ENFORCED, - ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.not_null: ConstraintSupport.NOT_SUPPORTED, ConstraintType.unique: ConstraintSupport.NOT_SUPPORTED, ConstraintType.primary_key: ConstraintSupport.NOT_SUPPORTED, ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, @@ -321,20 +323,6 @@ def get_rows_different_sql( return sql - @classmethod - def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]: - - constraint_expression = constraint.expression or '' - - rendered_column_constraint = None - if constraint.type == ConstraintType.check and constraint_expression: - rendered_column_constraint = f"CHECK ({constraint_expression})" - - if rendered_column_constraint: - rendered_column_constraint = rendered_column_constraint.strip() - - return rendered_column_constraint - def update_column_sql( self, dst_name: str, @@ -398,11 +386,10 @@ def format_columns(self, columns) -> List[Dict]: @classmethod def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List: rendered_columns = [] - for v in raw_columns.values(): rendered_columns.append(f"{quote_identifier(v['name'])} {v['data_type']}") if v.get("constraints"): - raise DbtRuntimeError("ClickHouse does not support column level constraints") + warn_or_error(ConstraintNotSupported(constraint='column', adapter='clickhouse')) return rendered_columns diff --git a/tests/integration/adapter/clickhouse/test_clickhouse_sql_header.py b/tests/integration/adapter/clickhouse/test_clickhouse_sql_header.py new file mode 100644 index 00000000..0b135a96 --- /dev/null +++ b/tests/integration/adapter/clickhouse/test_clickhouse_sql_header.py @@ -0,0 +1,28 @@ +import pytest +from dbt.tests.util import run_dbt_and_capture + +my_model_sql_header_sql = """ +{{ + config( + materialized = "table", + ) +}} + +{% call set_sql_header(config) %} +set log_comment = 'TEST_LOG_COMMENT'; +{%- endcall %} +select getSettings('log_comment') as column_name +""" + + +class TestSQLHeader: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_sql_header.sql": my_model_sql_header_sql, + } + + def test__sql_header(self, project): + _, log_output = run_dbt_and_capture(["run", "-s", "my_model_sql_header"], expect_pass=False) + + assert 'Multi-statements' in log_output diff --git a/tests/integration/adapter/constraints/fixtures_contraints.py b/tests/integration/adapter/constraints/fixtures_contraints.py index 1e0f858c..ee8988d2 100644 --- a/tests/integration/adapter/constraints/fixtures_contraints.py +++ b/tests/integration/adapter/constraints/fixtures_contraints.py @@ -1,4 +1,4 @@ -model_schema_yml = """ +contract_model_schema_yml = """ version: 2 models: - name: my_model @@ -135,3 +135,83 @@ 1::UInt32 as id, toDate('2019-01-01') as date_day """ + + +my_model_incremental_wrong_order_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 'blue' as color, + 1::UInt32 as id, + toDate('2019-01-01') as date_day +""" + +my_model_incremental_wrong_name_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 'blue' as color, + 1 as error, + '2019-01-01' as date_day +""" + +constraint_model_schema_yml = """ +version: 2 +models: + - name: bad_column_constraint_model + materialized: table + config: + contract: + enforced: true + columns: + - name: id + data_type: Int32 + constraints: + - type: check + expression: '> 0' + - name: color + data_type: String + - name: date_day + data_type: Date + - name: bad_foreign_key_model + config: + contract: + enforced: true + constraints: + - type: foreign_key + columns: [ id ] + expression: 'foreign_key_model (id)' + columns: + - name: id + data_type: Int32 +""" + +bad_column_constraint_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +SELECT 5::Int32 as id, 'black' as color, toDate('2023-01-01') as date_day +""" + +bad_foreign_key_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +SELECT 1::Int32 as id +""" diff --git a/tests/integration/adapter/constraints/test_constraints.py b/tests/integration/adapter/constraints/test_constraints.py index 3d9316aa..dafb1a67 100644 --- a/tests/integration/adapter/constraints/test_constraints.py +++ b/tests/integration/adapter/constraints/test_constraints.py @@ -1,9 +1,14 @@ import pytest from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file from fixtures_contraints import ( + bad_column_constraint_model_sql, + bad_foreign_key_model_sql, + constraint_model_schema_yml, + contract_model_schema_yml, model_data_type_schema_yml, - model_schema_yml, my_model_data_type_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_sql, my_model_view_wrong_name_sql, my_model_view_wrong_order_sql, my_model_wrong_name_sql, @@ -11,7 +16,7 @@ ) -class ClickHouseConstraintsColumnsEqual: +class ClickHouseContractColumnsEqual: """ dbt should catch these mismatches during its "preflight" checks. """ @@ -29,7 +34,7 @@ def data_types(self): ["'1'::Float64", "Float64", "Float64"], ] - def test__constraints_wrong_column_order(self, project): + def test__contract_wrong_column_order(self, project): # This no longer causes an error, since we enforce yaml column order run_dbt(["run", "-s", "my_model_wrong_order"], expect_pass=True) manifest = get_manifest(project.project_root) @@ -39,7 +44,7 @@ def test__constraints_wrong_column_order(self, project): assert contract_actual_config.enforced is True - def test__constraints_wrong_column_names(self, project): + def test__contract_wrong_column_names(self, project): _, log_output = run_dbt_and_capture(["run", "-s", "my_model_wrong_name"], expect_pass=False) run_dbt(["run", "-s", "my_model_wrong_name"], expect_pass=False) manifest = get_manifest(project.project_root) @@ -52,7 +57,7 @@ def test__constraints_wrong_column_names(self, project): expected = ["id", "error", "missing in definition", "missing in contract"] assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) - def test__constraints_wrong_column_data_types(self, project, data_types): + def test__contract_wrong_column_data_types(self, project, data_types): for (sql_column_value, schema_data_type, error_data_type) in data_types: # Write parametrized data_type to sql file write_file( @@ -63,7 +68,7 @@ def test__constraints_wrong_column_data_types(self, project, data_types): write_file( model_data_type_schema_yml.format(data_type='Int128'), "models", - "constraints_schema.yml", + "contract_schema.yml", ) results, log_output = run_dbt_and_capture( @@ -83,7 +88,7 @@ def test__constraints_wrong_column_data_types(self, project, data_types): ] assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) - def test__constraints_correct_column_data_types(self, project, data_types): + def test__contract_correct_column_data_types(self, project, data_types): for (sql_column_value, schema_data_type, _) in data_types: # Write parametrized data_type to sql file write_file( @@ -95,7 +100,7 @@ def test__constraints_correct_column_data_types(self, project, data_types): write_file( model_data_type_schema_yml.format(data_type=schema_data_type), "models", - "constraints_schema.yml", + "contract_schema.yml", ) run_dbt(["run", "-s", "my_model_data_type"]) @@ -108,43 +113,61 @@ def test__constraints_correct_column_data_types(self, project, data_types): assert contract_actual_config.enforced is True -class TestTableConstraintsColumnsEqual(ClickHouseConstraintsColumnsEqual): +class TestTableContractColumnsEqual(ClickHouseContractColumnsEqual): @pytest.fixture(scope="class") def models(self): return { "my_model_wrong_order.sql": my_model_wrong_order_sql, "my_model_wrong_name.sql": my_model_wrong_name_sql, - "constraints_schema.yml": model_schema_yml, + "contract_schema.yml": contract_model_schema_yml, } -class TestViewConstraintsColumnsEqual(ClickHouseConstraintsColumnsEqual): +class TestViewContractColumnsEqual(ClickHouseContractColumnsEqual): @pytest.fixture(scope="class") def models(self): return { "my_model_wrong_order.sql": my_model_view_wrong_order_sql, "my_model_wrong_name.sql": my_model_view_wrong_name_sql, - "constraints_schema.yml": model_schema_yml, + "contract_schema.yml": contract_model_schema_yml, } -# class TestViewConstraintsColumnsEqual(BaseViewConstraintsColumnsEqual): -# pass -# -# -# class TestIncrementalConstraintsColumnsEqual(BaseIncrementalConstraintsColumnsEqual): -# pass -# -# -# class TestTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): -# pass -# -# -# class TestTableConstraintsRollback(BaseConstraintsRollback): -# pass -# -# -# class TestIncrementalConstraintsRuntimeDdlEnforcement( -# BaseIncrementalConstraintsRuntimeDdlEnforcement -# ): -# pass +class TestIncrementalContractColumnsEqual(ClickHouseContractColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "contract_schema.yml": contract_model_schema_yml, + } + + +class TestBadConstraints: + @pytest.fixture(scope="class") + def models(self): + return { + "bad_column_constraint_model.sql": bad_column_constraint_model_sql, + "bad_foreign_key_model.sql": bad_foreign_key_model_sql, + "constraints_schema.yml": constraint_model_schema_yml, + } + + def test_invalid_column_constraint(self, project): + _, log_output = run_dbt_and_capture( + ["run", "-s", "bad_column_constraint_model"], expect_pass=True + ) + assert "not supported" in log_output + + def test_invalid_fk_constraint(self, project): + _, log_output = run_dbt_and_capture( + ["run", "-s", "bad_foreign_key_model"], expect_pass=True + ) + assert "not supported" in log_output + + +class TestModelConstrains: + @pytest.fixture(scope="class") + def models(self): + return { + "constrains_schema.yml": constraint_model_schema_yml, + }