From 09e076f7203ffe1486f3bf8394ec3bb9c5b38ddc Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Fri, 26 Apr 2024 10:27:27 -0600 Subject: [PATCH] Mwallace/merge 1.7 to main (#175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "Upgraded to dbt-core 1.4. (#146)" This reverts commit 3db05eb15d351d050caa52e50a589b1b82bcd6e2. * Fix version bumping * Migration bump2version (unmaintained) to bump-my-version * Import `bumpversion.cfg` from `dbt-snowflake` * Convert `bumpversion.cfg` to `bumpversion.toml` * Fix a few incorrect version labels * Remove macos and windows as integration targets * Fix requirements conflict between dbt 1.2 and tox * Bump version to 1.3 * Upgraded to dbt-core 1.4. (#146) * Upgraded to dbt-core 1.4. * Updated CHANGELOG. * Fixed policy fields definitions for mariadb and mysql5. * Replaced deprecated dbt.exceptions.raise_compiler_error() with dbt.exceptions.CompilationError. * Now using dbt.exceptions.DbtDatabaseError insead of dbt.exceptions.DatabaseException. * Update version * Update changelog --------- Co-authored-by: Doug Beatty * Bump version to 1.5 * Update dbt-core to 1.5 and implement support for model contracts (#163) Update dbt-core to 1.5.9 and implement support for constraints * Update changelog * Fix unit test error - The error was: `AttributeError: 'Namespace' object has no attribute 'MACRO_DEBUGGING'` * Allow Unix socket connection rather than just TCP (#165) * Support Black & MyPy pre-commit hooks (#167) * Add black and mypy as pre-commit hooks * Run black formatter on all files * Add MyPy configuration & make tweaks and ignore errors to make MyPy pass * Add .git-blame-ignore-revs to ignore `black` changes in git blame * Update changelog * Run black formatter on all files * Add black commit to .git-blame-ignore-revs * Run black formatter on all files * Add black commit to .git-blame-ignore-revs * Fix MyPy & black errors * Make the `database` config field nullable again * Fix mypy * Bump version to 1.6 * Bump version to 1.7 * Fix import error * Fix unicode decode error (#169) * Update setup.py Explicitly defining utf-8 encoding to avoid UnicodeDecodeError on setup.py in Windows environment * Update CHANGELOG.md --------- Co-authored-by: Scott Gunn <99990724+sagunn-echo@users.noreply.github.com> * Pin jsonschema version to fix tests (#174) * Add collation support in profiles.yml / Fix incorrect collation for utf8mb4 (#173) * Add support for charset and collation in profile.yml * Change mysql-connect-python version * Update readme with new profile options * Update setup.py with less restricting mysql-connector-python version after code review Co-authored-by: Matthew Wallace * Add collation and charset support for mariadb and mysql5 --------- Co-authored-by: Matthew Wallace * Pin jsonschema version to fix tests * Update black to resolve security alert https://github.com/dbeatty10/dbt-mysql/security/dependabot/5 --------- Co-authored-by: Luke Co-authored-by: Doug Beatty Co-authored-by: Scott Gunn <99990724+sagunn-echo@users.noreply.github.com> Co-authored-by: Krzysztof Zajączkowski <65259263+kzajaczkowski@users.noreply.github.com> --- CHANGELOG.md | 2 ++ README.md | 6 +++++- dbt/adapters/mariadb/connections.py | 7 +++++++ dbt/adapters/mysql/connections.py | 7 +++++++ dbt/adapters/mysql5/connections.py | 7 +++++++ dev-requirements.txt | 3 ++- setup.py | 4 ++-- 7 files changed, 32 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a58f452..5e2bf92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,14 @@ ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) +- Fix UnicodeDecodeErorr on setup.py ([#160](https://github.com/dbeatty10/dbt-mysql/issues/160)) ### Contributors - [@lpezet](https://github.com/lpezet) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@moszutij](https://github.com/moszutij) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146), [#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) - [@wesen](https://github.com/wesen) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) - [@mwallace582](https://github.com/mwallace582) ([#162](https://github.com/dbeatty10/dbt-mysql/pull/162), [#163](https://github.com/dbeatty10/dbt-mysql/pull/163), [#164](https://github.com/dbeatty10/dbt-mysql/issues/164), [#138](https://github.com/dbeatty10/dbt-mysql/issues/138)) +- [@sagunn-echo](https://github.com/sagunn-echo) ([#160](https://github.com/dbeatty10/dbt-mysql/issues/160)) ## dbt-mysql 1.1.0 (Feb 5, 2023) diff --git a/README.md b/README.md index b901ce4..5a305a7 100644 --- a/README.md +++ b/README.md @@ -86,17 +86,21 @@ your_profile_name: username: your_mysql_username password: your_mysql_password ssl_disabled: True + charset: utf8mb4 + collation: utf8mb4_0900_ai_ci ``` | Option | Description | Required? | Example | | --------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------ | ---------------------------------------------- | -| type | The specific adapter to use | Required | `mysql`, `mysql5` or `mariadb` | +| type | The specific adapter to use | Required | `mysql`, `mysql5` or `mariadb` | | server | The server (hostname) to connect to | Required | `yourorg.mysqlhost.com` | | port | The port to use | Optional | `3306` | | schema | Specify the schema (database) to build models into | Required | `analytics` | | username | The username to use to connect to the server | Required | `dbt_admin` | | password | The password to use for authenticating to the server | Required | `correct-horse-battery-staple` | | ssl_disabled | Set to enable or disable TLS connectivity to mysql5.x | Optional | `True` or `False` | +| charset | Specify charset to be used by a connection | Optional | `utf8mb4` | +| collation | Set to enable or disable TLS connectivity to mysql5.x | Optional | `utf8mb4_0900_ai_ci` | ### Notes diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index 94c9428..6e463de 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -26,6 +26,7 @@ class MariaDBCredentials(Credentials): password: Optional[str] = None charset: Optional[str] = None ssl_disabled: Optional[bool] = None + collation: Optional[str] = None _ALIASES = { "UID": "username", @@ -98,6 +99,12 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port + if credentials.charset: + kwargs["charset"] = credentials.charset + + if credentials.collation: + kwargs["collation"] = credentials.collation + try: connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 353d5ce..0e2af00 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -25,6 +25,7 @@ class MySQLCredentials(Credentials): username: Optional[str] = None password: Optional[str] = None charset: Optional[str] = None + collation: Optional[str] = None _ALIASES = { "UID": "username", @@ -94,6 +95,12 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port + if credentials.charset: + kwargs["charset"] = credentials.charset + + if credentials.collation: + kwargs["collation"] = credentials.collation + try: connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index 0160bd9..ee7cbc5 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -26,6 +26,7 @@ class MySQLCredentials(Credentials): password: Optional[str] = None charset: Optional[str] = None ssl_disabled: Optional[bool] = None + collation: Optional[str] = None _ALIASES = { "UID": "username", @@ -98,6 +99,12 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port + if credentials.charset: + kwargs["charset"] = credentials.charset + + if credentials.collation: + kwargs["collation"] = credentials.collation + try: connection.handle = mysql.connector.connect(**kwargs) connection.state = "open" diff --git a/dev-requirements.txt b/dev-requirements.txt index 161b0a0..70ec428 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,9 +5,10 @@ git+https://github.com/dbt-labs/dbt-core.git@1.7.latest#egg=dbt-tests-adapter&su # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor -black~=22.12 +black~=24.3 bumpversion~=0.6.0 ddtrace~=2.3 +jsonschema<=4.17 flake8~=6.1 flaky~=3.7 freezegun~=1.3 diff --git a/setup.py b/setup.py index f257d6f..dbc957f 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # pull long description from README this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, "README.md")) as f: +with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f: long_description = f.read() @@ -68,7 +68,7 @@ def _get_dbt_core_version(): include_package_data=True, install_requires=[ "dbt-core~={}".format(dbt_core_version), - "mysql-connector-python>=8.0.0,<8.1", + "mysql-connector-python>=8.0.0", ], zip_safe=False, classifiers=[