Merge branch 'main' into test_various_snapshot_configs

.changes/unreleased/Breaking Changes-20241016-185117.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Drop support for Python 3.8
+time: 2024-10-16T18:51:17.581547-04:00
+custom:
+  Author: mikealfare
+  Issue: "1373"

.changes/unreleased/Dependencies-20240724-040744.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Update wheel requirement from ~=0.42 to ~=0.43"
+time: 2024-07-24T04:07:44.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 1304

.changes/unreleased/Features-20240505-011838.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: add is_retryable test case when raise ServiceUnavailable
+time: 2024-05-05T01:18:38.737882+09:00
+custom:
+  Author: jx2lee
+  Issue: "682"

.changes/unreleased/Features-20240911-234859.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Adds the ability to set optional `quota_project` in profile
+time: 2024-09-11T23:48:59.767649+01:00
+custom:
+    Author: jcarpenter12
+    Issue: 1343 1344

.changes/unreleased/Fixes-20241028-172719.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: use "direct" write for non-partitioned python model materializations
+time: 2024-10-28T17:27:19.306348-07:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "1318"

.changes/unreleased/Under the Hood-20240910-212052.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Isolating distribution testing
+time: 2024-09-10T21:20:52.574204-04:00
+custom:
+    Author: leahwicz
+    Issue: "1290"

.changes/unreleased/Under the Hood-20241104-173815.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Separate credentials functionality into its own module for reuse in retry and
+  python submissions
+time: 2024-11-04T17:38:15.940962-05:00
+custom:
+  Author: mikealfare
+  Issue: "1391"

.changes/unreleased/Under the Hood-20241107-143856.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Create a retry factory to simplify retry strategies across dbt-bigquery
+time: 2024-11-07T14:38:56.210445-05:00
+custom:
+  Author: mikealfare osalama
+  Issue: "1395"

.github/scripts/integration-test-matrix.js

@@ -1,6 +1,6 @@
 module.exports = ({ context }) => {
-  const defaultPythonVersion = "3.8";
-  const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"];
+  const defaultPythonVersion = "3.9";
+  const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"];
   const supportedAdapters = ["bigquery"];
 
   // if PR, generate matrix based on files changed and PR labels
@@ -44,7 +44,7 @@ module.exports = ({ context }) => {
 
             if (labels.includes("test macos") || testAllLabel) {
               include.push({
-                os: "macos-12",
+                os: "macos-14",
                 adapter,
                 "python-version": pythonVersion,
               });
@@ -78,7 +78,7 @@ module.exports = ({ context }) => {
   // additionally include runs for all adapters, on macos and windows,
   // but only for the default python version
   for (const adapter of supportedAdapters) {
-    for (const operatingSystem of ["windows-latest", "macos-12"]) {
+    for (const operatingSystem of ["windows-latest", "macos-14"]) {
       include.push({
         os: operatingSystem,
         adapter: adapter,

.github/workflows/integration.yml

@@ -33,6 +33,7 @@ on:
   # all PRs, important to note that `pull_request_target` workflows
   # will run in the context of the target branch of a PR
   pull_request_target:
+    types: [opened, reopened, synchronize, labeled]
   # manual trigger
   workflow_dispatch:
     inputs:
@@ -280,10 +281,10 @@ jobs:
           persist-credentials: false
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install python dependencies
         run: |

.github/workflows/main.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -70,7 +70,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     env:
       TOXENV: "unit"
@@ -127,7 +127,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -163,7 +163,7 @@ jobs:
           overwrite: true
 
   test-build:
-    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }}
 
     if: needs.build.outputs.is_alpha == 0
 
@@ -174,19 +174,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest, macos-14, windows-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+        dist-type: ["whl", "gz"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install python dependencies
         run: |
           python -m pip install --user --upgrade pip
-          python -m pip install --upgrade wheel setuptools twine check-wheel-contents
+          python -m pip install --upgrade wheel
           python -m pip --version
+
       - uses: actions/download-artifact@v4
         with:
           name: dist
@@ -195,15 +198,10 @@ jobs:
       - name: Show distributions
         run: ls -lh dist/
 
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check wheel distributions
+      - name: Install ${{ matrix.dist-type }} distributions
         run: |
-          python -c "import dbt.adapters.bigquery"
-      - name: Install source distributions
-        run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check source distributions
+          find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+
+      - name: Check ${{ matrix.dist-type }} distributions
         run: |
           python -c "import dbt.adapters.bigquery"

.pre-commit-config.yaml

@@ -24,7 +24,6 @@ repos:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311

CONTRIBUTING.md

@@ -54,7 +54,7 @@ To confirm you have the correct version of `dbt-core` installed please run `dbt
 
 ### Initial Setup
 
-`dbt-bigquery` contains [unit](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/unit) and [integration](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/integration) tests. Integration tests require testing against an actual BigQuery warehouse. We have CI set up to test against a BigQuery warehouse. In order to run integration tests locally, you will need a `test.env` file in the root of the repository that contains credentials for BigQuery.
+`dbt-bigquery` contains [unit](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/functional) tests. functional tests require testing against an actual BigQuery warehouse. We have CI set up to test against a BigQuery warehouse. In order to run functional tests locally, you will need a `test.env` file in the root of the repository that contains credentials for BigQuery.
 
 Note: This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing. To create your `test.env` file, copy the provided example file, then supply your relevant credentials.
 
@@ -67,7 +67,7 @@ $EDITOR test.env
 There are a few methods for running tests locally.
 
 #### `tox`
-`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.8, Python 3.9, Python 3.10, and Python 3.11 in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py38`. The configuration of these tests are located in `tox.ini`.
+`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.9, Python 3.10, and Python 3.11 in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py39`. The configuration of these tests are located in `tox.ini`.
 
 #### `pytest`
 Finally, you can also run a specific test or group of tests using `pytest` directly. With a Python virtualenv active and dev dependencies installed you can do things like:
@@ -104,6 +104,6 @@ You don't need to worry about which `dbt-bigquery` version your change will go i
 
 dbt Labs provides a CI environment to test changes to the `dbt-bigquery` adapter and periodic checks against the development version of `dbt-core` through Github Actions.
 
-A `dbt-bigquery` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
+A `dbt-bigquery` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or functional test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
 
 Once all tests are passing, you have updated the changelog to reflect and tag your issue/pr for reference with a small description of the change, and your PR has been approved, a `dbt-bigquery` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:

dbt/adapters/bigquery/__init__.py

@@ -1,8 +1,8 @@
-from dbt.adapters.bigquery.connections import BigQueryConnectionManager  # noqa
-from dbt.adapters.bigquery.connections import BigQueryCredentials
-from dbt.adapters.bigquery.relation import BigQueryRelation  # noqa
-from dbt.adapters.bigquery.column import BigQueryColumn  # noqa
-from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget, PartitionConfig  # noqa
+from dbt.adapters.bigquery.column import BigQueryColumn
+from dbt.adapters.bigquery.connections import BigQueryConnectionManager
+from dbt.adapters.bigquery.credentials import BigQueryCredentials
+from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget, PartitionConfig
+from dbt.adapters.bigquery.relation import BigQueryRelation
 
 from dbt.adapters.base import AdapterPlugin
 from dbt.include import bigquery

dbt/adapters/bigquery/clients.py

@@ -0,0 +1,69 @@
+from google.api_core.client_info import ClientInfo
+from google.api_core.client_options import ClientOptions
+from google.api_core.retry import Retry
+from google.auth.exceptions import DefaultCredentialsError
+from google.cloud.bigquery import Client as BigQueryClient
+from google.cloud.dataproc_v1 import BatchControllerClient, JobControllerClient
+from google.cloud.storage import Client as StorageClient
+
+from dbt.adapters.events.logging import AdapterLogger
+
+import dbt.adapters.bigquery.__version__ as dbt_version
+from dbt.adapters.bigquery.credentials import (
+    BigQueryCredentials,
+    create_google_credentials,
+    set_default_credentials,
+)
+
+
+_logger = AdapterLogger("BigQuery")
+
+
+def create_bigquery_client(credentials: BigQueryCredentials) -> BigQueryClient:
+    try:
+        return _create_bigquery_client(credentials)
+    except DefaultCredentialsError:
+        _logger.info("Please log into GCP to continue")
+        set_default_credentials()
+        return _create_bigquery_client(credentials)
+
+
+@Retry()  # google decorator. retries on transient errors with exponential backoff
+def create_gcs_client(credentials: BigQueryCredentials) -> StorageClient:
+    return StorageClient(
+        project=credentials.execution_project,
+        credentials=create_google_credentials(credentials),
+    )
+
+
+@Retry()  # google decorator. retries on transient errors with exponential backoff
+def create_dataproc_job_controller_client(credentials: BigQueryCredentials) -> JobControllerClient:
+    return JobControllerClient(
+        credentials=create_google_credentials(credentials),
+        client_options=ClientOptions(api_endpoint=_dataproc_endpoint(credentials)),
+    )
+
+
+@Retry()  # google decorator. retries on transient errors with exponential backoff
+def create_dataproc_batch_controller_client(
+    credentials: BigQueryCredentials,
+) -> BatchControllerClient:
+    return BatchControllerClient(
+        credentials=create_google_credentials(credentials),
+        client_options=ClientOptions(api_endpoint=_dataproc_endpoint(credentials)),
+    )
+
+
+@Retry()  # google decorator. retries on transient errors with exponential backoff
+def _create_bigquery_client(credentials: BigQueryCredentials) -> BigQueryClient:
+    return BigQueryClient(
+        credentials.execution_project,
+        create_google_credentials(credentials),
+        location=getattr(credentials, "location", None),
+        client_info=ClientInfo(user_agent=f"dbt-bigquery-{dbt_version.version}"),
+        client_options=ClientOptions(quota_project_id=credentials.quota_project),
+    )
+
+
+def _dataproc_endpoint(credentials: BigQueryCredentials) -> str:
+    return f"{credentials.dataproc_region}-dataproc.googleapis.com:443"

dbt/adapters/bigquery/column.py

@@ -1,9 +1,10 @@
 from dataclasses import dataclass
-from typing import Optional, List, TypeVar, Iterable, Type, Any, Dict, Union
+from typing import Any, Dict, Iterable, List, Optional, Type, TypeVar, Union
+
+from google.cloud.bigquery import SchemaField
 
 from dbt.adapters.base.column import Column
 
-from google.cloud.bigquery import SchemaField
 
 _PARENT_DATA_TYPE_KEY = "__parent_data_type"