Merge branch 'main' into incremental-seed-hash

.changes/unreleased/Features-20230112-191705.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: ✨ add unix-style wildcard selector method
+time: 2023-01-12T19:17:05.841918-07:00
+custom:
+  Author: z3z1ma
+  Issue: "6598"

.changes/unreleased/Features-20230206-084749.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Enable diff based partial parsing
+time: 2023-02-06T08:47:49.688889-05:00
+custom:
+  Author: gshank
+  Issue: "6592"

.changes/unreleased/Features-20230222-130632.yaml

@@ -1,6 +1,6 @@
 kind: Features
-body: get_column_schema_from_query_macro
+body: Enforce contracts on models materialized as tables and views
 time: 2023-02-22T13:06:32.583743-05:00
 custom:
-  Author: jtcohen6 michelleark
-  Issue: "6751"
+  Author: jtcohen6 michelleark emmyoop
+  Issue: 6751 7034

.github/workflows/main.yml

@@ -55,10 +55,9 @@ jobs:
           python -m pip --version
           python -m pip install pre-commit
           pre-commit --version
-          python -m pip install mypy==0.942
-          mypy --version
           python -m pip install -r requirements.txt
           python -m pip install -r dev-requirements.txt
+          mypy --version
           dbt --version
 
       - name: Run pre-commit hooks

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
     alias: flake8-check
     stages: [manual]
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.942
+  rev: v0.981
   hooks:
   - id: mypy
     # N.B.: Mypy is... a bit fragile.

core/dbt/clients/system.py

@@ -18,7 +18,6 @@
 from dbt.events.functions import fire_event
 from dbt.events.types import (
     SystemCouldNotWrite,
-    SystemErrorRetrievingModTime,
     SystemExecutingCmd,
     SystemStdOut,
     SystemStdErr,
@@ -77,11 +76,7 @@ def find_matching(
                 relative_path = os.path.relpath(absolute_path, absolute_path_to_search)
                 relative_path_to_root = os.path.join(relative_path_to_search, relative_path)
 
-                modification_time = 0.0
-                try:
-                    modification_time = os.path.getmtime(absolute_path)
-                except OSError:
-                    fire_event(SystemErrorRetrievingModTime(path=absolute_path))
+                modification_time = os.path.getmtime(absolute_path)
                 if reobj.match(local_file) and (
                     not ignore_spec or not ignore_spec.match_file(relative_path_to_root)
                 ):

core/dbt/contracts/files.py

@@ -63,8 +63,6 @@ def absolute_path(self) -> str:
 
     @property
     def original_file_path(self) -> str:
-        # this is mostly used for reporting errors. It doesn't show the project
-        # name, should it?
         return os.path.join(self.searched_path, self.relative_path)
 
     def file_size(self) -> int:

core/dbt/events/types.proto

@@ -762,7 +762,18 @@ message FinishedRunningStatsMsg {
 
 // I - Project parsing
 
-// Skipping I001, I002, I003, I004, I005, I006, I007
+// I001
+message InputFileDiffError {
+    string category = 1;
+    string file_id = 2;
+}
+
+message InputFileDiffErrorMsg {
+    EventInfo info = 1;
+    InputFileDiffError data = 2;
+}
+
+// Skipping I002, I003, I004, I005, I006, I007
 
 // I008
 message InvalidValueForField {
@@ -1808,15 +1819,7 @@ message MainStackTraceMsg {
     MainStackTrace data = 2;
 }
 
-// Z004
-message SystemErrorRetrievingModTime {
-    string path = 1;
-}
-
-message SystemErrorRetrievingModTimeMsg {
-    EventInfo info = 1;
-    SystemErrorRetrievingModTime data = 2;
-}
+// skipping Z004
 
 // Z005
 message SystemCouldNotWrite {

core/dbt/events/types.py

@@ -768,7 +768,16 @@ def message(self) -> str:
 # =======================================================
 
 
-# Skipping I001, I002, I003, I004, I005, I006, I007
+@dataclass
+class InputFileDiffError(DebugLevel, pt.InputFileDiffError):
+    def code(self):
+        return "I001"
+
+    def message(self) -> str:
+        return f"Error processing file diff: {self.category}, {self.file_id}"
+
+
+# Skipping I002, I003, I004, I005, I006, I007
 
 
 @dataclass
@@ -1891,13 +1900,7 @@ def message(self) -> str:
         return self.stack_trace
 
 
-@dataclass
-class SystemErrorRetrievingModTime(ErrorLevel, pt.SystemErrorRetrievingModTime):
-    def code(self):
-        return "Z004"
-
-    def message(self) -> str:
-        return f"Error retrieving modification time for file {self.path}"
+# Skipped Z004
 
 
 @dataclass

core/dbt/graph/selector_methods.py

@@ -1,4 +1,5 @@
 import abc
+from fnmatch import fnmatch
 from itertools import chain
 from pathlib import Path
 from typing import Set, List, Dict, Iterator, Tuple, Any, Union, Type, Optional, Callable
@@ -46,10 +47,13 @@ class MethodName(StrEnum):
     Metric = "metric"
     Result = "result"
     SourceStatus = "source_status"
+    Wildcard = "wildcard"
 
 
-def is_selected_node(fqn: List[str], node_selector: str):
-
+def is_selected_node(
+    fqn: List[str],
+    node_selector: str,
+) -> bool:
     # If qualified_name exactly matches model name (fqn's leaf), return True
     if fqn[-1] == node_selector:
         return True
@@ -59,15 +63,26 @@ def is_selected_node(fqn: List[str], node_selector: str):
     if len(flat_fqn) < len(node_selector.split(".")):
         return False
 
+    slurp_from_ix: Optional[int] = None
     for i, selector_part in enumerate(node_selector.split(".")):
-        # if we hit a GLOB, then this node is selected
-        if selector_part == SELECTOR_GLOB:
-            return True
+        if any(wildcard in selector_part for wildcard in ("*", "?", "[", "]")):
+            slurp_from_ix = i
+            break
         elif flat_fqn[i] == selector_part:
             continue
         else:
             return False
 
+    if slurp_from_ix is not None:
+        # If we have a wildcard, we need to make sure that the selector matches the
+        # rest of the fqn, this is 100% backwards compatible with the old behavior of
+        # encountering a wildcard but more expressive in naturally allowing you to
+        # match the rest of the fqn with more advanced patterns
+        return fnmatch(
+            ".".join(flat_fqn[slurp_from_ix:]),
+            ".".join(node_selector.split(".")[slurp_from_ix:]),
+        )
+
     # if we get all the way down here, then the node is a match
     return True
 
@@ -195,7 +210,7 @@ class TagSelectorMethod(SelectorMethod):
     def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
         """yields nodes from included that have the specified tag"""
         for node, real_node in self.all_nodes(included_nodes):
-            if selector in real_node.tags:
+            if any(fnmatch(tag, selector) for tag in real_node.tags):
                 yield node
 
 
@@ -213,7 +228,7 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
         parts = selector.split(".")
         target_package = SELECTOR_GLOB
         if len(parts) == 1:
-            target_source, target_table = parts[0], None
+            target_source, target_table = parts[0], SELECTOR_GLOB
         elif len(parts) == 2:
             target_source, target_table = parts
         elif len(parts) == 3:
@@ -228,13 +243,12 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
             raise DbtRuntimeError(msg)
 
         for node, real_node in self.source_nodes(included_nodes):
-            if target_package not in (real_node.package_name, SELECTOR_GLOB):
+            if not fnmatch(real_node.package_name, target_package):
                 continue
-            if target_source not in (real_node.source_name, SELECTOR_GLOB):
+            if not fnmatch(real_node.source_name, target_source):
                 continue
-            if target_table not in (None, real_node.name, SELECTOR_GLOB):
+            if not fnmatch(real_node.name, target_table):
                 continue
-
             yield node
 
 
@@ -255,9 +269,9 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
             raise DbtRuntimeError(msg)
 
         for node, real_node in self.exposure_nodes(included_nodes):
-            if target_package not in (real_node.package_name, SELECTOR_GLOB):
+            if not fnmatch(real_node.package_name, target_package):
                 continue
-            if target_name not in (real_node.name, SELECTOR_GLOB):
+            if not fnmatch(real_node.name, target_name):
                 continue
 
             yield node
@@ -280,9 +294,9 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
             raise DbtRuntimeError(msg)
 
         for node, real_node in self.metric_nodes(included_nodes):
-            if target_package not in (real_node.package_name, SELECTOR_GLOB):
+            if not fnmatch(real_node.package_name, target_package):
                 continue
-            if target_name not in (real_node.name, SELECTOR_GLOB):
+            if not fnmatch(real_node.name, target_name):
                 continue
 
             yield node
@@ -306,15 +320,15 @@ class FileSelectorMethod(SelectorMethod):
     def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
         """Yields nodes from included that match the given file name."""
         for node, real_node in self.all_nodes(included_nodes):
-            if Path(real_node.original_file_path).name == selector:
+            if fnmatch(Path(real_node.original_file_path).name, selector):
                 yield node
 
 
 class PackageSelectorMethod(SelectorMethod):
     def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
         """Yields nodes from included that have the specified package"""
         for node, real_node in self.all_nodes(included_nodes):
-            if real_node.package_name == selector:
+            if fnmatch(real_node.package_name, selector):
                 yield node
 
 
@@ -395,7 +409,7 @@ class TestNameSelectorMethod(SelectorMethod):
     def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
         for node, real_node in self.parsed_nodes(included_nodes):
             if real_node.resource_type == NodeType.Test and hasattr(real_node, "test_metadata"):
-                if real_node.test_metadata.name == selector:  # type: ignore[union-attr]
+                if fnmatch(real_node.test_metadata.name, selector):  # type: ignore[union-attr]
                     yield node
 
 

core/dbt/include/global_project/macros/materializations/models/view/create_view_as.sql

@@ -16,7 +16,11 @@
   {%- set sql_header = config.get('sql_header', none) -%}
 
   {{ sql_header if sql_header is not none }}
-  create view {{ relation }} as (
+  create view {{ relation }}
+    {%- if config.get('contract', False) %}
+      {{ get_assert_columns_equivalent(sql) }}
+    {%- endif %}
+  as (
     {{ sql }}
   );
 {%- endmacro %}

core/dbt/parser/base.py

@@ -320,7 +320,7 @@ def update_parsed_node_config(
                 original_file_path = parsed_node.original_file_path
                 error_message = "\n    `contract=true` can only be configured within `schema.yml` files\n      NOT within a model file(ex: .sql, .py) or `dbt_project.yml`."
                 raise ParsingError(
-                    f"Original File Path: ({original_file_path})\nConstraints must be defined in a `yml` schema configuration file like `schema.yml`.\nOnly the SQL table materialization is supported for constraints. \n`data_type` values must be defined for all columns and NOT be null or blank.{error_message}"
+                    f"Original File Path: ({original_file_path})\nConstraints must be defined in a `yml` schema configuration file like `schema.yml`.\nOnly the SQL table and view materializations are supported for constraints. \n`data_type` values must be defined for all columns and NOT be null or blank.{error_message}"
                 )
 
         # unrendered_config is used to compare the original database/schema/alias