diff --git a/.changes/unreleased/Fixes-20240925-154514.yaml b/.changes/unreleased/Fixes-20240925-154514.yaml new file mode 100644 index 00000000000..8db05cc6a0e --- /dev/null +++ b/.changes/unreleased/Fixes-20240925-154514.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Pass test user config to adapter pre_hook by explicitly adding test builder config to node +time: 2024-09-25T15:45:14.459598-07:00 +custom: + Author: colin-rogers-dbt + Issue: "10484" diff --git a/core/dbt/context/context_config.py b/core/dbt/context/context_config.py index a5b38bb415e..b1aace475c7 100644 --- a/core/dbt/context/context_config.py +++ b/core/dbt/context/context_config.py @@ -6,10 +6,12 @@ from dbt.adapters.factory import get_config_class_by_name from dbt.config import IsFQNResource, Project, RuntimeConfig from dbt.contracts.graph.model_config import get_config_for +from dbt.exceptions import SchemaConfigError from dbt.flags import get_flags from dbt.node_types import NodeType from dbt.utils import fqn_search from dbt_common.contracts.config.base import BaseConfig, merge_config_dicts +from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import DbtInternalError @@ -237,8 +239,12 @@ def calculate_node_config_dict( base=base, patch_config_dict=patch_config_dict, ) - finalized = config.finalize_and_validate() - return finalized.to_dict(omit_none=True) + try: + finalized = config.finalize_and_validate() + return finalized.to_dict(omit_none=True) + except ValidationError as exc: + # we got a ValidationError - probably bad types in config() + raise SchemaConfigError(exc, node=config) from exc class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]): diff --git a/core/dbt/parser/schema_generic_tests.py b/core/dbt/parser/schema_generic_tests.py index 14e2dbc862a..9f2538f06f0 100644 --- a/core/dbt/parser/schema_generic_tests.py +++ b/core/dbt/parser/schema_generic_tests.py @@ -200,7 +200,9 @@ def parse_generic_test( # this is the ContextConfig that is used in render_update config: ContextConfig = self.initial_config(fqn) - + # Adding the builder's config to the ContextConfig + # is needed to ensure the config makes it to the pre_model hook which dbt-snowflake needs + config.add_config_call(builder.config) # builder.args contains keyword args for the test macro, # not configs which have been separated out in the builder. # The keyword args are not completely rendered until compilation. diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py index 2bbfac85c5c..952b951e507 100644 --- a/tests/functional/configs/test_configs.py +++ b/tests/functional/configs/test_configs.py @@ -2,6 +2,7 @@ import pytest +from dbt.exceptions import SchemaConfigError from dbt.tests.util import ( check_relations_equal, run_dbt, @@ -81,7 +82,7 @@ def test_tests_materialization_proj_config(self, project): tests_dir = os.path.join(project.project_root, "tests") write_file("select * from foo", tests_dir, "test.sql") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() @@ -93,7 +94,7 @@ def test_seeds_materialization_proj_config(self, project): seeds_dir = os.path.join(project.project_root, "seeds") write_file("id1, id2\n1, 2", seeds_dir, "seed.csv") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() @@ -107,7 +108,7 @@ def test_seeds_materialization_schema_config(self, project): ) write_file("id1, id2\n1, 2", seeds_dir, "myseed.csv") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() diff --git a/tests/functional/configs/test_disabled_model.py b/tests/functional/configs/test_disabled_model.py index 23cf8fde1e0..a918067ac15 100644 --- a/tests/functional/configs/test_disabled_model.py +++ b/tests/functional/configs/test_disabled_model.py @@ -1,8 +1,7 @@ import pytest -from dbt.exceptions import CompilationError, ParsingError +from dbt.exceptions import CompilationError, ParsingError, SchemaConfigError from dbt.tests.util import get_manifest, run_dbt -from dbt_common.dataclass_schema import ValidationError from tests.functional.configs.fixtures import ( my_model, my_model_2, @@ -394,8 +393,8 @@ def models(self): "my_model.sql": my_model, } - def test_invalis_config(self, project): - with pytest.raises(ValidationError) as exc: + def test_invalid_config(self, project): + with pytest.raises(SchemaConfigError) as exc: run_dbt(["parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "'True and False' is not of type 'boolean'" diff --git a/tests/functional/schema_tests/data_test_config.py b/tests/functional/schema_tests/data_test_config.py index 377f14aac04..a6a16818000 100644 --- a/tests/functional/schema_tests/data_test_config.py +++ b/tests/functional/schema_tests/data_test_config.py @@ -2,6 +2,8 @@ import pytest +from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.graph.nodes import TestNode from dbt.exceptions import CompilationError from dbt.tests.util import get_manifest, run_dbt from tests.functional.schema_tests.fixtures import ( @@ -10,14 +12,44 @@ same_key_error_yml, seed_csv, table_sql, + test_custom_color_from_config, ) +def _select_test_node(manifest: Manifest, pattern: re.Pattern[str]): + # Find the test_id dynamically + test_id = None + for node_id in manifest.nodes: + if pattern.match(node_id): + test_id = node_id + break + + # Ensure the test_id was found + assert test_id is not None, "Test ID matching the pattern was not found in the manifest nodes" + return manifest.nodes[test_id] + + +def get_table_persistence(project, table_name): + sql = f""" + SELECT + relpersistence + FROM pg_class + WHERE relname like '%{table_name}%' + """ + result = project.run_sql(sql, fetch="one") + assert len(result) == 1 + return result[0] + + class BaseDataTestsConfig: @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seed_csv} + @pytest.fixture(scope="class") + def macros(self): + return {"custom_color_from_config.sql": test_custom_color_from_config} + @pytest.fixture(scope="class") def project_config_update(self): return { @@ -35,29 +67,24 @@ def models(self): return {"table.sql": table_sql, "custom_config.yml": custom_config_yml} def test_custom_config(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) + run_dbt(["run"]) + run_dbt(["test", "--log-level", "debug"], expect_pass=False) + manifest = get_manifest(project.project_root) # Pattern to match the test_id without the specific suffix pattern = re.compile(r"test\.test\.accepted_values_table_color__blue__red\.\d+") - # Find the test_id dynamically - test_id = None - for node_id in manifest.nodes: - if pattern.match(node_id): - test_id = node_id - break - - # Ensure the test_id was found - assert ( - test_id is not None - ), "Test ID matching the pattern was not found in the manifest nodes" - + test_node: TestNode = _select_test_node(manifest, pattern) # Proceed with the assertions - test_node = manifest.nodes[test_id] assert "custom_config_key" in test_node.config assert test_node.config["custom_config_key"] == "some_value" + # pattern = re.compile(r"test\.test\.custom_color_from_config.*") + # test_node = _select_test_node(manifest, pattern) + persistence = get_table_persistence(project, "custom_color_from_config_table_color") + + assert persistence == "u" + class TestMixedDataTestConfig(BaseDataTestsConfig): @pytest.fixture(scope="class") diff --git a/tests/functional/schema_tests/fixtures.py b/tests/functional/schema_tests/fixtures.py index bf16148e0c7..ac266545712 100644 --- a/tests/functional/schema_tests/fixtures.py +++ b/tests/functional/schema_tests/fixtures.py @@ -1285,6 +1285,12 @@ values: ['blue', 'red'] config: custom_config_key: some_value + - custom_color_from_config: + severity: error + config: + test_color: orange + store_failures: true + unlogged: True """ mixed_config_yml = """ @@ -1296,9 +1302,13 @@ data_tests: - accepted_values: values: ['blue', 'red'] - severity: warn config: custom_config_key: some_value + severity: warn + - custom_color_from_config: + severity: error + config: + test_color: blue """ same_key_error_yml = """ @@ -1327,9 +1337,16 @@ 8,green,80 9,yellow,90 10,blue,100 -""" +""".strip() table_sql = """ -- content of the table.sql select * from {{ ref('seed') }} """ + +test_custom_color_from_config = """ +{% test custom_color_from_config(model, column_name) %} + select * from {{ model }} + where color = '{{ config.get('test_color') }}' +{% endtest %} +"""