Merge pull request #512 from dbt-labs/tox

tox testing

.circleci/config.yml

@@ -13,11 +13,11 @@ jobs:
       - run:
           name: "Run Tests - Postgres"
           environment:
-            POSTGRES_TEST_HOST: localhost
-            POSTGRES_TEST_USER: root
-            POSTGRES_TEST_PASS: ''
-            POSTGRES_TEST_PORT: 5432
-            POSTGRES_TEST_DBNAME: circle_test
+            POSTGRES_HOST: localhost
+            POSTGRES_USER: root
+            DBT_ENV_SECRET_POSTGRES_PASS: ''
+            POSTGRES_PORT: 5432
+            POSTGRES_DATABASE: circle_test
           command: ./run_test.sh postgres
       - store_artifacts:
           path: ./integration_tests/logs

.github/workflows/ci.yml

@@ -0,0 +1,49 @@
+# **what?**
+# Run tests for dbt-utils against supported adapters
+
+# **why?**
+# To ensure that dbt-utils works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+
+jobs:
+  run-tests:
+      uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+      with:
+        # no need to pass postgres vars in.  We can just use the defaults in the local container
+        # redshift
+        REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+        REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+        REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+        REDSHIFT_SCHEMA: "dpe_integration_tests_redshift_${{ github.run_number }}"
+        REDSHIFT_PORT: 5439
+        # bigquery
+        BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+        BIGQUERY_SCHEMA: "dpe_integration_tests_bigquery_${{ github.run_number }}"
+        # snowflake
+        SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
+        SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
+        SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
+        SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
+        SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
+        # databricks
+        DATABRICKS_SCHEMA: "integration_tests_databricks_${{ github.run_number }}"
+        DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
+        DATABRICKS_HTTP_PATH: ${{ vars.DATABRICKS_HTTP_PATH }}
+      secrets:
+        DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
+        BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
+        SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+        DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
+        DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DBT_ENV_SECRET_DATABRICKS_TOKEN }}

.github/workflows/local_only.yml

@@ -0,0 +1,83 @@
+# **what?**
+# Run tests for packages not supported for cloud testing
+#
+# **why?**
+# To ensure that packages works as expected with all supported adapters
+
+# **when?**
+# On push, PR or manually called
+
+
+name: Package Integration Tests - Local Only
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+env:
+    PYTHON_VERSION: "3.11"
+    POSTGRES_HOST: "localhost"
+    POSTGRES_USER: "root"
+    POSTGRES_PORT: "5432"
+    POSTGRES_DATABASE: "postgres_test"
+    DBT_ENV_SECRET_POSTGRES_PASS: "password"  # this isn't actually a secret since it only runs on the runner
+
+jobs:
+    run-tests:
+        runs-on: ubuntu-latest
+        services:
+            postgres:
+                image: postgres
+                env:
+                    POSTGRES_USER: ${{ env.POSTGRES_USER }}
+                    POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }}
+                    POSTGRES_DB: ${{ env.POSTGRES_DATABASE }}
+                    POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
+                options: >-
+                    --health-cmd pg_isready
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 5432:5432
+        strategy:
+            fail-fast: false
+            matrix:
+                # these adapters are tested in this repo but are not tested as part of the dbt Cloud images.
+                # This list should include anything not listed in supported_adapters.env
+                adapter: [duckdb, postgres]
+
+        steps:
+            - name: "Checkout ${{ github.event.repository }} "
+              uses: actions/checkout@v4
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install ${{ matrix.adapter }}"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install dbt-${{ matrix.adapter }}
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Run integration tests with tox on ${{ matrix.adapter }}"
+              run: |
+                  tox -e dbt_integration_${{ matrix.adapter }}
+              env:
+                # postgres
+                POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
+                POSTGRES_USER: ${{ env.POSTGRES_USER }}
+                DBT_ENV_SECRET_POSTGRES_PASS: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }}
+                POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
+                POSTGRES_DATABASE: ${{ env.POSTGRES_DATABASE }}
+                POSTGRES_SCHEMA: "integration_tests_postgres_${{ github.run_number }}"
+                # duckdb - needs no vars

.gitignore

@@ -3,7 +3,8 @@
 venv/
 
 # Environment variables
-.env
+# .env
+test.env
 
 # DBT artifacts
 target/
@@ -13,7 +14,7 @@ dbt_packages/
 integration_tests/state/
 site/
 env/
-profiles.yml
+# profiles.yml
 package-lock.yml
 
 # IDE

integration_tests/profiles.yml

@@ -0,0 +1,84 @@
+
+# HEY! This file is used in the integration tests with CI.
+# You should __NEVER__ check credentials into version control. Thanks for reading :)
+
+config:
+    send_anonymous_usage_stats: False
+    use_colors: True
+
+integration_tests:
+  target: postgres
+  outputs:
+    postgres:
+      type: postgres
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
+      port: "{{ env_var('POSTGRES_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_DATABASE') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}"
+      threads: 5
+
+    redshift:
+      type: redshift
+      host: "{{ env_var('REDSHIFT_HOST') }}"
+      user: "{{ env_var('REDSHIFT_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
+      port: "{{ env_var('REDSHIFT_PORT') | as_number }}"
+      schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}"
+      threads: 5
+
+    bigquery:
+      type: bigquery
+      method: service-account-json
+      keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}"
+      project: "{{ env_var('BIGQUERY_PROJECT') }}"
+      dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}"
+      threads: 10
+
+    snowflake:
+      type: snowflake
+      account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_USER') }}"
+      password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
+      role: "{{ env_var('SNOWFLAKE_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
+      schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}"
+      threads: 10
+
+    databricks:
+      type: databricks
+      schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}"
+      host: "{{ env_var('DATABRICKS_HOST') }}"
+      http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}"
+      token: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_TOKEN') }}"
+      threads: 10
+
+    duckdb:
+      type: duckdb
+      path: ./duck.db
+
+    trino:
+      type: trino
+      host: "{{ env_var('TRINO_TEST_HOST') }}"
+      port: "{{ env_var('TRINO_TEST_PORT') | as_number }}"
+      method: ldap
+      user: "{{ env_var('TRINO_TEST_USER') }}"
+      password: "{{ env_var('TRINO_TEST_PASS') }}"
+      catalog: "{{ env_var('TRINO_TEST_CATALOG_NAME') }}"
+      schema: dbt_project_evaluator_integration_tests_trino
+      threads: 5
+      session_properties:
+        query_max_stage_count: 275
+
+    clickhouse:
+      type: clickhouse
+      host: "{{ env_var('CLICKHOUSE_TEST_HOST') }}"
+      port: "{{ env_var('CLICKHOUSE_TEST_PORT') | as_number }}"
+      user: "{{ env_var('CLICKHOUSE_TEST_USER') }}"
+      password: "{{ env_var('CLICKHOUSE_TEST_PASS') }}"
+      dbname: "{{ env_var('CLICKHOUSE_TEST_DBNAME') }}"
+      schema: dbt_project_evaluator_integration_tests_clickhouse
+      threads: 5