diff --git a/.circleci/config.yml b/.circleci/config.yml index 19147aca..4c920012 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,11 +13,11 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_TEST_HOST: localhost - POSTGRES_TEST_USER: root - POSTGRES_TEST_PASS: '' - POSTGRES_TEST_PORT: 5432 - POSTGRES_TEST_DBNAME: circle_test + POSTGRES_HOST: localhost + POSTGRES_USER: root + DBT_ENV_SECRET_POSTGRES_PASS: '' + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test command: ./run_test.sh postgres - store_artifacts: path: ./integration_tests/logs diff --git a/.github/workflows/local_only.yml b/.github/workflows/local_only.yml new file mode 100644 index 00000000..2449cb3d --- /dev/null +++ b/.github/workflows/local_only.yml @@ -0,0 +1,83 @@ +# **what?** +# Run tests for packages not supported for cloud testing +# +# **why?** +# To ensure that packages works as expected with all supported adapters + +# **when?** +# On push, PR or manually called + + +name: Package Integration Tests - Local Only + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +env: + PYTHON_VERSION: "3.11" + POSTGRES_HOST: "localhost" + POSTGRES_USER: "root" + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: "postgres_test" + DBT_ENV_SECRET_POSTGRES_PASS: "password" # this isn't actually a secret since it only runs on the runner + +jobs: + run-tests: + runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_DB: ${{ env.POSTGRES_DATABASE }} + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + strategy: + fail-fast: false + matrix: + # these adapters are tested in this repo but are not tested as part of the dbt Cloud images. + # This list should include anything not listed in supported_adapters.env + adapter: [duckdb, postgres] + + steps: + - name: "Checkout ${{ github.event.repository }} " + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install ${{ matrix.adapter }}" + run: | + python -m pip install --upgrade pip + pip install dbt-${{ matrix.adapter }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Run integration tests with tox on ${{ matrix.adapter }}" + run: | + tox -e dbt_integration_${{ matrix.adapter }} + env: + # postgres + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + DBT_ENV_SECRET_POSTGRES_PASS: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_PORT: ${{ env.POSTGRES_PORT }} + POSTGRES_DATABASE: ${{ env.POSTGRES_DATABASE }} + POSTGRES_SCHEMA: "integration_tests_postgres_${{ github.run_number }}" + # duckdb - needs no vars diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 50e2a231..f4b73176 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the integration tests with CircleCI. +# HEY! This file is used in the integration tests with CI. # You should __NEVER__ check credentials into version control. Thanks for reading :) config: @@ -16,7 +16,7 @@ integration_tests: pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" port: "{{ env_var('POSTGRES_PORT') | as_number }}" dbname: "{{ env_var('POSTGRES_DATABASE') }}" - schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_test') }}" + schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}" threads: 5 redshift: @@ -26,7 +26,7 @@ integration_tests: pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" dbname: "{{ env_var('REDSHIFT_DATABASE') }}" port: "{{ env_var('REDSHIFT_PORT') | as_number }}" - schema: "{{ env_var('REDSHIFT_SCHEMA') }}" + schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}" threads: 5 bigquery: @@ -35,7 +35,7 @@ integration_tests: keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" - dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" threads: 10 snowflake: @@ -46,7 +46,7 @@ integration_tests: role: "{{ env_var('SNOWFLAKE_ROLE') }}" database: "{{ env_var('SNOWFLAKE_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" - schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}" threads: 10 databricks: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests_2/profiles.yml similarity index 50% rename from integration_tests/ci/sample.profiles.yml rename to integration_tests_2/profiles.yml index dff42772..f4b73176 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests_2/profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the integration tests with CircleCI. +# HEY! This file is used in the integration tests with CI. # You should __NEVER__ check credentials into version control. Thanks for reading :) config: @@ -11,46 +11,47 @@ integration_tests: outputs: postgres: type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: dbt_project_evaluator_integration_tests_postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}" threads: 5 redshift: type: redshift - host: "{{ env_var('REDSHIFT_TEST_HOST') }}" - user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" - dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" - port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: dbt_project_evaluator_integration_tests_redshift + host: "{{ env_var('REDSHIFT_HOST') }}" + user: "{{ env_var('REDSHIFT_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" + dbname: "{{ env_var('REDSHIFT_DATABASE') }}" + port: "{{ env_var('REDSHIFT_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}" threads: 5 bigquery: type: bigquery method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: dbt_project_evaluator_integration_tests_bigquery + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" threads: 10 snowflake: type: snowflake - account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" - role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: dbt_project_evaluator_integration_tests_snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}" threads: 10 databricks: type: databricks - schema: dbt_project_evaluator_integration_tests_databricks + schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" host: "{{ env_var('DATABRICKS_TEST_HOST') }}" http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" diff --git a/run_test.sh b/run_test.sh index 07b39521..c7e9ed4f 100755 --- a/run_test.sh +++ b/run_test.sh @@ -13,12 +13,6 @@ fi . $VENV cd integration_tests - -if [[ ! -e ~/.dbt/profiles.yml ]]; then - mkdir -p ~/.dbt - cp ci/sample.profiles.yml ~/.dbt/profiles.yml -fi - dbt deps --target $1 || exit 1 dbt build -x --target $1 --full-refresh || exit 1 diff --git a/run_tox_tests.sh b/run_tox_tests.sh new file mode 100755 index 00000000..9b80e093 --- /dev/null +++ b/run_tox_tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "Running tests for the first project" +cd integration_tests +dbt deps --target $1 || exit 1 +dbt build -x --target $1 --full-refresh || exit 1 + +echo "Running tests for the second project" +cd ../integration_tests_2 +dbt deps --target $1 || exit 1 +dbt seed --full-refresh --target $1 || exit 1 +dbt run -x --target $1 --full-refresh || exit 1 diff --git a/tox.ini b/tox.ini index 90738ef7..24e1d645 100644 --- a/tox.ini +++ b/tox.ini @@ -34,104 +34,50 @@ passenv = # Snowflake integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_snowflake] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target snowflake - dbt build -x --target snowflake --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target snowflake - dbt seed --full-refresh --target snowflake - dbt run -x --target snowflake --full-refresh - + bash ./run_tox_tests.sh snowflake # Postgres integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_postgres] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target postgres - dbt build -x --target postgres --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target postgres - dbt seed --full-refresh --target postgres - dbt run -x --target postgres --full-refresh + bash ./run_tox_tests.sh postgres # BigQuery integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_bigquery] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target bigquery - dbt build -x --target bigquery --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target bigquery - dbt seed --full-refresh --target bigquery - dbt run -x --target bigquery --full-refresh + bash ./run_tox_tests.sh bigquery # redshift integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_redshift] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target redshift - dbt build -x --target redshift --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target redshift - dbt seed --full-refresh --target redshift - dbt run -x --target redshift --full-refresh + bash ./run_tox_tests.sh redshift +# note that duckdb is not a supported dbt target for dbt Cloud testing [testenv:dbt_integration_duckdb] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target duckdb - dbt build -x --target duckdb --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target duckdb - dbt seed --full-refresh --target duckdb - dbt run -x --target duckdb --full-refresh + bash ./run_tox_tests.sh duckdb [testenv:dbt_integration_databricks] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target duckdb - dbt build -x --target duckdb --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target duckdb - dbt seed --full-refresh --target duckdb - dbt run -x --target duckdb --full-refresh \ No newline at end of file + bash ./run_tox_tests.sh databricks \ No newline at end of file