diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0c2b95d..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,94 +0,0 @@ -version: 2.1 - -jobs: - unit: - environment: - DBT_INVOCATION_ENV: circle - docker: - - image: fishtownanalytics/test-container:10 - steps: - - checkout - - run: tox -e flake8,unit - - integration-8-0: - environment: - DBT_INVOCATION_ENV: circle - DBT_MYSQL_SERVER_NAME: mysql-server-8-0 - docker: - - image: fishtownanalytics/test-container:10 - - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: dbt - name: mysql-server-8-0 - steps: - - checkout - - run: - name: Wait for MySQL - command: dockerize -wait tcp://mysql-server-8-0:3306 -timeout 5m -wait-retry-interval 5s - - run: - name: Run integration tests - command: tox -e integration-mysql-8.0 - no_output_timeout: 1h - - store_artifacts: - path: ./logs - - integration-5-7: - environment: - DBT_INVOCATION_ENV: circle - DBT_MYSQL_SERVER_NAME: mysql-server-5-7 - docker: - - image: fishtownanalytics/test-container:10 - - image: mysql:5.7 - environment: - MYSQL_ROOT_PASSWORD: dbt - command: [--explicit-defaults-for-timestamp=ON] - name: mysql-server-5-7 - steps: - - checkout - - run: - name: Wait for MySQL - command: dockerize -wait tcp://mysql-server-5-7:3306 -timeout 5m -wait-retry-interval 5s - - run: - name: Run integration tests - command: tox -e integration-mysql-5.7 - no_output_timeout: 1h - - store_artifacts: - path: ./logs - - integration-mariadb-10-5: - environment: - DBT_INVOCATION_ENV: circle - DBT_MYSQL_SERVER_NAME: mariadb-server-10-5 - docker: - - image: fishtownanalytics/test-container:10 - - image: mariadb:10.5 - environment: - MYSQL_ROOT_PASSWORD: dbt - command: [--explicit-defaults-for-timestamp=ON] - name: mariadb-server-10-5 - steps: - - checkout - - run: - name: Wait for MariaDB - command: dockerize -wait tcp://mariadb-server-10-5:3306 -timeout 5m -wait-retry-interval 5s - - run: - name: Run integration tests - command: tox -e integration-mariadb-10.5 - no_output_timeout: 1h - - store_artifacts: - path: ./logs - -workflows: - version: 2 - test-everything: - jobs: - - unit - - integration-8-0: - requires: - - unit - - integration-5-7: - requires: - - unit - - integration-mariadb-10-5: - requires: - - unit diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..da7e039 --- /dev/null +++ b/.flake8 @@ -0,0 +1,16 @@ +[flake8] +select = + E + W + F +ignore = + # makes Flake8 work like black + W503, + W504, + # makes Flake8 work like black + E203, + E741, + E501, +exclude = tests +per-file-ignores = + */__init__.py: F401 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..7f7d15b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# This codeowners file is used to ensure all PRs require reviews from the appropriate users. + +* @dbeatty10 @mwallace582 diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js new file mode 100644 index 0000000..5f6d93f --- /dev/null +++ b/.github/scripts/integration-test-matrix.js @@ -0,0 +1,105 @@ +module.exports = ({ context }) => { + const defaultPythonVersion = "3.8"; + const supportedPythonVersions = ["3.8", "3.9"]; + const supportedAdapters = ["mysql", "mysql5", "mariadb"]; + const adapterImages = { + mysql: "mysql:8.0", + mysql5: "mysql:5.7", + mariadb: "mariadb:10.5", + }; + + // if PR, generate matrix based on files changed and PR labels + if (context.eventName.includes("pull_request")) { + // `changes` is a list of adapter names that have related + // file changes in the PR + // ex: ['postgres', 'snowflake'] + const changes = JSON.parse(process.env.CHANGES); + const labels = context.payload.pull_request.labels.map(({ name }) => name); + console.log("labels", labels); + console.log("changes", changes); + const testAllLabel = labels.includes("test all"); + const include = []; + + for (const adapter of supportedAdapters) { + if ( + changes.includes(adapter) || + testAllLabel || + labels.includes(`test ${adapter}`) + ) { + for (const pythonVersion of supportedPythonVersions) { + if ( + pythonVersion === defaultPythonVersion || + labels.includes(`test python${pythonVersion}`) || + testAllLabel + ) { + // always run tests on ubuntu by default + include.push({ + os: "ubuntu-latest", + adapter, + "database-image": adapterImages[adapter], + "python-version": pythonVersion, + }); + + if (labels.includes("test windows") || testAllLabel) { + include.push({ + os: "windows-latest", + adapter, + "database-image": adapterImages[adapter], + "python-version": pythonVersion, + }); + } + + if (labels.includes("test macos") || testAllLabel) { + include.push({ + os: "macos-latest", + adapter, + "database-image": adapterImages[adapter], + "python-version": pythonVersion, + }); + } + } + } + } + } + + console.log("matrix", { include }); + + return { + include, + }; + } + // if not PR, generate matrix of python version, adapter, and operating + // system to run integration tests on + + const include = []; + // run for all adapters and python versions on ubuntu + for (const adapter of supportedAdapters) { + for (const pythonVersion of supportedPythonVersions) { + include.push({ + os: 'ubuntu-latest', + adapter: adapter, + "database-image": adapterImages[adapter], + "python-version": pythonVersion, + }); + } + } + + // additionally include runs for all adapters, on macos and windows, + // but only for the default python version + for (const adapter of supportedAdapters) { + for (const operatingSystem of ["windows-latest", "macos-latest"]) { + include.push({ + os: operatingSystem, + adapter: adapter, + "database-image": adapterImages[adapter], + "python-version": defaultPythonVersion, + }); + } + } + + console.log("matrix", { include }); + + return { + include, + }; +}; diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh new file mode 100755 index 0000000..d28a40c --- /dev/null +++ b/.github/scripts/update_dbt_core_branch.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +set -e + +git_branch=$1 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" +postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" +tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$postgres_req_sed_pattern" $target_req_file + sed -i "" "$tests_req_sed_pattern" $target_req_file +else + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$postgres_req_sed_pattern" $target_req_file + sed -i "$tests_req_sed_pattern" $target_req_file +fi +core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) +bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..00d56b4 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,238 @@ +# **what?** +# This workflow runs all integration tests for supported OS +# and python versions and core adapters. If triggered by PR, +# the workflow will only run tests for adapters related +# to code changes. Use the `test all` and `test ${adapter}` +# label to run all or additional tests. Use `ok to test` +# label to mark PRs from forked repositories that are safe +# to run integration tests for. Requires secrets to run +# against different warehouses. + +# **why?** +# This checks the functionality of dbt from a user's perspective +# and attempts to catch functional regressions. + +# **when?** +# This workflow will run on every push to a protected branch +# and when manually triggered. It will also run for all PRs, including +# PRs from forks. The workflow will be skipped until there is a label +# to mark the PR as safe to run. + +name: Adapter Integration Tests + +on: + # pushes to release branches + push: + branches: + - "main" + - "develop" + - "*.latest" + - "releases/*" + # all PRs, important to note that `pull_request_target` workflows + # will run in the context of the target branch of a PR + pull_request_target: + # manual trigger + workflow_dispatch: + inputs: + dbt-core-branch: + description: "branch of dbt-core to use in dev-requirements.txt" + required: false + type: string + +# explicitly turn off permissions for `GITHUB_TOKEN` +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +# sets default shell to bash, for all operating systems +defaults: + run: + shell: bash + +jobs: + # generate test metadata about what files changed and the testing matrix to use + test-metadata: + # run if not a PR from a forked repository or has a label to mark as safe to test + if: >- + github.event_name != 'pull_request_target' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.generate-matrix.outputs.result }} + + steps: + - name: Check out the repository (non-PR) + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if relevant files changed + if: github.event_name == 'pull_request_target' + # https://github.com/marketplace/actions/paths-changes-filter + # For each filter, it sets output variable named by the filter to the text: + # 'true' - if any of changed files matches any of filter rules + # 'false' - if none of changed files matches any of filter rules + # also, returns: + # `changes` - JSON array with names of all filters matching any of the changed files + uses: dorny/paths-filter@v2 + id: get-changes + with: + token: ${{ secrets.GITHUB_TOKEN }} + filters: | + mysql: + - 'dbt/include/mysql/**' + - 'dbt/adapters/mysql/**' + - 'tests/**' + - 'dev-requirements.txt' + mysql5: + - 'dbt/include/mysql5/**' + - 'dbt/adapters/mysql5/**' + - 'tests/**' + - 'dev-requirements.txt' + mariadb: + - 'dbt/include/mariadb/**' + - 'dbt/adapters/mariadb/**' + - 'tests/**' + - 'dev-requirements.txt' + + - name: Generate integration test matrix + id: generate-matrix + uses: actions/github-script@v6 + env: + CHANGES: ${{ steps.get-changes.outputs.changes }} + with: + script: | + const script = require('./.github/scripts/integration-test-matrix.js') + const matrix = script({ context }) + console.log(matrix) + return matrix + + test: + name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} + + # run if not a PR from a forked repository or has a label to mark as safe to test + # also checks that the matrix generated is not empty + if: >- + needs.test-metadata.outputs.matrix && + fromJSON( needs.test-metadata.outputs.matrix ).include[0] && + ( + github.event_name != 'pull_request_target' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'ok to test') + ) + runs-on: ${{ matrix.os }} + needs: test-metadata + + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} + + env: + TOXENV: integration-${{ matrix.adapter }} + PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + services: + database: + image: ${{ matrix.database-image }} + env: + MYSQL_ROOT_PASSWORD: dbt + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + # GitHub Actions do not support passing command line flags to services + # https://github.com/orgs/community/discussions/26688#discussioncomment-3252882 + - name: Set mariadb database systems variables + if: ${{ contains(fromJSON('["mysql5", "mariadb"]'), matrix.adapter) }} + run: | + docker exec ${{ job.services.database.id }} sh -c 'MYSQL_PWD=dbt mysql -u root --execute "SET GLOBAL explicit_defaults_for_timestamp = ON;"' + + - name: Check out the repository + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + + # explicity checkout the branch for the PR, + # this is necessary for the `pull_request_target` event + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Update dev_requirements.txt + if: inputs.dbt-core-branch != '' + run: | + pip install bumpversion + ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + + - name: Run tox + run: tox -- --ddtrace + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: logs + path: ./logs + + - name: Get current date + if: always() + id: date + run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv + path: integration_results.csv + + require-label-comment: + runs-on: ubuntu-latest + + needs: test + + permissions: + pull-requests: write + + steps: + - name: Needs permission PR comment + if: >- + needs.test.result == 'skipped' && + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "You do not have permissions to run integration tests. The maintainers of this repo "\ + "need to label this PR with `ok to test` in order to run integration tests!" + check_for_duplicate_msg: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..2d3db96 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,212 @@ +# **what?** +# Runs code quality checks, unit tests, and verifies python build on +# all code commited to the repository. This workflow should not +# require any secrets since it runs for PRs from forked repos. +# By default, secrets are not passed to workflows running from +# a forked repo. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to a release +# branch, and when manually triggered. + +name: Tests and Code Checks + +on: + push: + branches: + - "main" + - "develop" + - "*.latest" + - "releases/*" + pull_request: + workflow_dispatch: + +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + code-quality: + name: code-quality + + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install -r dev-requirements.txt + python -m pip --version + pre-commit --version + dbt --version + + - name: Run pre-commit hooks + run: pre-commit run --all-files --show-diff-on-failure + + unit: + name: unit test / python ${{ matrix.python-version }} + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.9'] + + env: + TOXENV: "unit" + PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Run tox + run: tox + + - name: Get current date + if: always() + id: date + run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv + path: unit_results.csv + + build: + name: build packages + + runs-on: ubuntu-latest + + outputs: + is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install --upgrade setuptools wheel twine check-wheel-contents + python -m pip --version + + - name: Build distributions + run: ./scripts/build-dist.sh + + - name: Show distributions + run: ls -lh dist/ + + - name: Check distribution descriptions + run: | + twine check dist/* + + - name: Check wheel contents + run: | + check-wheel-contents dist/*.whl --ignore W002,W007,W008 + + - name: Check if this is an alpha version + id: check-is-alpha + run: | + export is_alpha=0 + if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi + echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v3 + with: + name: dist + path: dist/ + + test-build: + name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} + + if: needs.build.outputs.is_alpha == 0 + + needs: build + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.8', '3.9'] + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install --upgrade wheel + python -m pip --version + + - uses: actions/download-artifact@v3 + with: + name: dist + path: dist/ + + - name: Show distributions + run: ls -lh dist/ + + - name: Install wheel distributions + run: | + find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check wheel distributions + run: | + dbt --version + + - name: Install source distributions + run: | + find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check source distributions + run: | + dbt --version diff --git a/CHANGELOG.md b/CHANGELOG.md index d79f1bf..824421a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features - Support dbt v1.4 ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) +- Migrate CircleCI to GitHub Actions ([#120](https://github.com/dbeatty10/dbt-mysql/issues/120)) ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) @@ -10,6 +11,7 @@ - [@lpezet](https://github.com/lpezet) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) +- [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162)) ## dbt-mysql 1.1.0 (Feb 5, 2023) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd78b31..3fe2d53 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ Ready to contribute? [Here's](https://jarv.is/notes/how-to-pull-request-fork-git python3 -m venv env source env/bin/activate python3 -m pip install --upgrade pip - python3 -m pip install -r requirements-dev.txt -r requirements-editable.txt + python3 -m pip install -e . -r dev-requirements.txt pre-commit install source env/bin/activate ``` diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d6aa5b5 --- /dev/null +++ b/Makefile @@ -0,0 +1,56 @@ +.DEFAULT_GOAL:=help + +.PHONY: dev +dev: ## Installs adapter in develop mode along with development dependencies + @\ + pip install -e . -r dev-requirements.txt && pre-commit install + +.PHONY: dev-uninstall +dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment + ## Useful when updating versions, or if you accidentally installed into the system interpreter + pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y + pip uninstall -y dbt-mysql + +.PHONY: flake8 +flake8: ## Runs flake8 against staged changes to enforce style guide. + @\ + pre-commit run --hook-stage manual flake8-check | grep -v "INFO" + +.PHONY: lint +lint: ## Runs flake8 code checks against staged changes. + @\ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; + +.PHONY: linecheck +linecheck: ## Checks for all Python lines 100 characters or more + @\ + find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; + +.PHONY: unit +unit: ## Runs unit tests with py38. + @\ + tox -e py38 + +.PHONY: test +test: ## Runs unit tests with py38 and code checks against staged changes. + @\ + tox -p -e py38; \ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; + +.PHONY: integration +integration: ## Runs mysql integration tests with py38. + @\ + tox -e py38-mysql --; \ + tox -e py38-mysql5 --; \ + tox -e py38-mariadb --; + +.PHONY: clean + @echo "cleaning repo" + @git clean -f -X + +.PHONY: help +help: ## Show this help message. + @echo 'usage: make [target]' + @echo + @echo 'targets:' + @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 10d8e7e..b901ce4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # dbt-mysql +[![Tests and Code Checks](https://github.com/dbeatty10/dbt-mysql/actions/workflows/main.yml/badge.svg)](https://github.com/dbeatty10/dbt-mysql/actions/workflows/main.yml) +[![Integration Tests Badge](https://github.com/dbeatty10/dbt-mysql/actions/workflows/integration.yml/badge.svg)](https://github.com/dbeatty10/dbt-mysql/actions/workflows/integration.yml) + This plugin ports [dbt](https://getdbt.com) functionality to MySQL and MariaDB. This is an experimental plugin: @@ -128,7 +131,7 @@ Additionally, many DBMS have relation names with three parts whereas MySQL has o ### Running Tests -See [test/README.md](test/README.md) for details on running the integration tests. +See [tests/README.md](tests/README.md) for details on running the integration tests. ### Reporting bugs and contributing code diff --git a/RELEASE.md b/RELEASE.md index 019ef1b..c423720 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -53,7 +53,7 @@ PyPI recognizes [pre-release versioning conventions](https://packaging.python.or ## Post-release 1. Create `{minor-version}.latest` branch. Example: - `git checkout -b 1.5.latest` - - Update the branch names in `requirements-dev.txt` from `@{previous-version}.latest` (or `@main`) to `@{minor-version}.latest` + - Update the branch names in `dev-requirements.txt` from `@{previous-version}.latest` (or `@main`) to `@{minor-version}.latest` - `git push` 1. Bump the version in `main` to be the next minor alpha. Example: - `git checkout main` @@ -61,6 +61,6 @@ PyPI recognizes [pre-release versioning conventions](https://packaging.python.or - `git checkout -b bump-1.6.0a1` - Minor releases: `bumpversion --no-tag --new-version 1.6.0a1 num` - - Update the branch names in `requirements-dev.txt` from `@{previous-version}.latest` to `@{minor-version}.latest` (or `@main`) + - Update the branch names in `dev-requirements.txt` from `@{previous-version}.latest` to `@{minor-version}.latest` (or `@main`) - Commit with message `Bump dbt-mysql 1.6.0a1` - `git push` diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..bb4f4d8 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,24 @@ +# install latest changes in dbt-core +# TODO: how to automate switching from develop to version branches? +# git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter + +# if version 1.x or greater -> pin to major version +# if version 0.x -> pin to minor +bumpversion~=0.6.0 +ddtrace~=2.3 +flake8~=6.1 +flaky~=3.7 +freezegun~=1.3 +ipdb~=0.13.13 +pre-commit~=3.5 +pre-commit-hooks~=4.5 +pytest~=7.4 +pytest-csv~=3.0 +pytest-dotenv~=0.5.2 +pytest-logbook~=1.2 +pytest-xdist~=3.5 +pytz~=2023.3 +tox~=4.11 +twine~=4.0 +wheel~=0.42 diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 4c54314..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,19 +0,0 @@ -# install latest changes in dbt-core -# TODO: how to automate switching from develop to version branches? -# git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter - -wheel -twine -freezegun==0.3.9 -pytest~=7.0 -mock>=1.3.0 -flake8>=3.5.0 -pytz==2017.2 -bumpversion==0.5.3 -tox==3.2.0 -ipdb -pytest-xdist>=2.1.0,<3 -flaky>=3.5.3,<4 -pytest-dotenv -pre-commit~=2.20.0 diff --git a/requirements-editable.txt b/requirements-editable.txt deleted file mode 100644 index d6e1198..0000000 --- a/requirements-editable.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh new file mode 100755 index 0000000..3c38083 --- /dev/null +++ b/scripts/build-dist.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eo pipefail + +DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )" + +PYTHON_BIN=${PYTHON_BIN:-python} + +echo "$PYTHON_BIN" + +set -x + +rm -rf "$DBT_PATH"/dist +rm -rf "$DBT_PATH"/build +mkdir -p "$DBT_PATH"/dist + +cd "$DBT_PATH" +$PYTHON_BIN setup.py sdist bdist_wheel + +set +x diff --git a/setup.py b/setup.py index 1441b2c..7c67449 100644 --- a/setup.py +++ b/setup.py @@ -1,81 +1,65 @@ #!/usr/bin/env python +import os import sys +import re +# require python 3.7 or newer if sys.version_info < (3, 7): print("Error: dbt does not support this version of Python.") print("Please upgrade to Python 3.7 or higher.") sys.exit(1) +# require version of setuptools that supports find_namespace_packages +from setuptools import setup + try: from setuptools import find_namespace_packages except ImportError: + # the user has a downlevel version of setuptools. print("Error: dbt requires setuptools v40.1.0 or higher.") - print( - 'Please upgrade setuptools with "pip install --upgrade setuptools" and try again' - ) + print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again") sys.exit(1) -from pathlib import Path -from setuptools import setup - - -# pull the long description from the README -README = Path(__file__).parent / "README.md" - - -# used for this adapter's version and in determining the compatible dbt-core version -VERSION = Path(__file__).parent / "dbt/adapters/mysql/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -def _core_patch(plugin_patch: str): - """ - Determines the compatible dbt-core patch given this plugin's patch - - Args: - plugin_patch: the version patch of this plugin - """ - pre_release_phase = "".join([i for i in plugin_patch if not i.isdigit()]) - if pre_release_phase: - if pre_release_phase not in ["a", "b", "rc"]: - raise ValueError(f"Invalid prerelease patch: {plugin_patch}") - return f"0{pre_release_phase}1" - return "0" +# pull long description from README +this_directory = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(this_directory, "README.md")) as f: + long_description = f.read() -# require a compatible minor version (~=) and prerelease if this is a prerelease -def _core_version(plugin_version: str = _plugin_version()) -> str: - """ - Determine the compatible dbt-core version give this plugin's version. +# get this package's version from dbt/adapters//__version__.py +def _get_plugin_version_dict(): + _version_path = os.path.join(this_directory, "dbt", "adapters", "mysql", "__version__.py") + _semver = r"""(?P\d+)\.(?P\d+)\.(?P\d+)""" + _pre = r"""((?Pa|b|rc)(?P
\d+))?"""
+    _nightly = r"""(\.(?P[a-z0-9]+)?)?"""
+    _version_pattern = rf"""version\s*=\s*["']{_semver}{_pre}{_nightly}["']"""
+    with open(_version_path) as f:
+        match = re.search(_version_pattern, f.read().strip())
+        if match is None:
+            raise ValueError(f"invalid version at {_version_path}")
+        return match.groupdict()
 
-    We assume that the plugin must agree with `dbt-core` down to the minor version.
 
-    Args:
-        plugin_version: the version of this plugin, this is an argument in case we ever want to unit test this
-    """
-    try:
-        major, minor, plugin_patch = plugin_version.split(".")
-    except ValueError:
-        raise ValueError(f"Invalid version: {plugin_version}")
+# require a compatible minor version (~=), prerelease if this is a prerelease
+def _get_dbt_core_version():
+    parts = _get_plugin_version_dict()
+    minor = "{major}.{minor}.0".format(**parts)
+    pre = parts["prekind"] + "1" if parts["prekind"] else ""
+    return f"{minor}{pre}"
 
-    return f"{major}.{minor}.{_core_patch(plugin_patch)}"
 
+package_name = "dbt-mysql"
+package_version = "1.4.0a1"
+dbt_core_version = _get_dbt_core_version()
+description = """The MySQL adapter plugin for dbt"""
 
 setup(
-    name="dbt-mysql",
-    version=_plugin_version(),
-    description="The MySQL adapter plugin for dbt",
-    long_description=README.read_text(),
+    name=package_name,
+    version=package_version,
+    description=description,
+    long_description=long_description,
     long_description_content_type="text/markdown",
     author="Doug Beatty",
     author_email="doug.beatty@gmail.com",
@@ -83,7 +67,7 @@ def _core_version(plugin_version: str = _plugin_version()) -> str:
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        f"dbt-core~={_core_version()}",
+        "dbt-core~={}".format(dbt_core_version),
         "mysql-connector-python>=8.0.0,<8.1",
     ],
     zip_safe=False,
diff --git a/tests/README.md b/tests/README.md
index fe8954c..a6dc376 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -17,7 +17,22 @@ PYTHONPATH=. pytest tests/functional/adapter/test_basic.py
 ## Full example
 
 ### Prerequisites
-- [`dbt-tests-adapter`](https://github.com/dbt-labs/dbt-core/tree/main/tests/adapter) package
+
+#### Dependencies
+
+`pip install -r ./dev-requirements.txt`
+
+#### Python Version
+
+Python 3.8 and 3.9 are supported test targets and may need to be installed before tests can run.
+
+##### Ubuntu
+
+```
+sudo add-apt-repository ppa:deadsnakes/ppa
+sudo apt-get update
+sudo apt-get install python3.8 python3.8-distutils python3.9 python3.9-distutils
+```
 
 ### Environment variables
 
@@ -87,22 +102,25 @@ sql_mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ALLOW_INVALID
 
 ### Run tests
 
-Run the test specs in this repository:
+Run all the tests via `make`:
 ```shell
-PYTHONPATH=. pytest -v --profile mysql tests/functional && \
-PYTHONPATH=. pytest -v --profile mysql5 tests/functional && \
-PYTHONPATH=. pytest -v --profile mariadb tests/functional
+make unit
+make integration
 ```
 
 Or run all the tests via `tox`:
 ```shell
-tox -e flake8,unit && \
-tox -e integration-mariadb-10.5 && \
-tox -e integration-mysql-5.7 && \
-tox -e integration-mysql-8.0
+tox
+```
+
+Or run the test specs directly
+```shell
+PYTHONPATH=. pytest -v --profile mysql tests/functional && \
+PYTHONPATH=. pytest -v --profile mysql5 tests/functional && \
+PYTHONPATH=. pytest -v --profile mariadb tests/functional
 ```
 
-Run a single test
+Or run a single test
 ```shell
 pytest -v --profile mysql tests/functional/adapter/test_basic.py::TestEmptyMySQL::test_empty
 ```
diff --git a/tox.ini b/tox.ini
index 2d8a5e5..03d2ca2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,42 +1,28 @@
 [tox]
 skipsdist = True
-envlist = flake8, unit, integration-mysql-8.0, integration-mysql-5.7
+envlist = py38,py39
 
-
-[testenv:flake8]
-basepython = python3.9
-commands = /bin/bash -c '$(which flake8) --select=E,W,F --max-line-length=88 --extend-ignore=E203,W504 dbt/'
-deps =
-     -r{toxinidir}/requirements-dev.txt
-
-[testenv:unit]
-basepython = python3.9
-commands = /bin/bash -c '{envpython} -m pytest -v {posargs} tests/unit'
-passenv = DBT_INVOCATION_ENV
-deps =
-   -r{toxinidir}/requirements-dev.txt
-   -r{toxinidir}/requirements-editable.txt
-
-[testenv:integration-mysql-8.0]
-basepython = python3.9
-commands = {envpython} -m pytest -v --profile mysql {posargs} tests/functional
-passenv = DBT_INVOCATION_ENV DBT_MYSQL_SERVER_NAME DBT_MYSQL_80_PORT
-deps =
-   -r{toxinidir}/requirements-dev.txt
-   -r{toxinidir}/requirements-editable.txt
-
-[testenv:integration-mysql-5.7]
-basepython = python3.9
-commands = {envpython} -m pytest -v --profile mysql5 {posargs} tests/functional
-passenv = DBT_INVOCATION_ENV DBT_MYSQL_SERVER_NAME DBT_MYSQL_57_PORT
+[testenv:{unit,py38,py39,py}]
+description = unit testing
+skip_install = true
+passenv =
+    DBT_*
+    PYTEST_ADDOPTS
+commands = {envpython} -m pytest {posargs} tests/unit
 deps =
-   -r{toxinidir}/requirements-dev.txt
-   -r{toxinidir}/requirements-editable.txt
+  -rdev-requirements.txt
+  -e.
 
-[testenv:integration-mariadb-10.5]
-basepython = python3.9
-commands = {envpython} -m pytest -v --profile mariadb {posargs} tests/functional
-passenv = DBT_INVOCATION_ENV DBT_MYSQL_SERVER_NAME DBT_MARIADB_105_PORT
+[testenv:{integration,py38,py39,py}-{mysql,mysql5,mariadb}]
+description = adapter plugin integration testing
+skip_install = true
+passenv =
+    DBT_*
+    PYTEST_ADDOPTS
+commands =
+  mysql: {envpython} -m pytest -v --profile mysql {posargs} tests/functional
+  mysql5: {envpython} -m pytest -v --profile mysql5 {posargs} tests/functional
+  mariadb: {envpython} -m pytest -v --profile mariadb {posargs} tests/functional
 deps =
-  -r{toxinidir}/requirements-dev.txt
-  -r{toxinidir}/requirements-editable.txt
+  -rdev-requirements.txt
+  -e.