Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Switch default driver to psycopg3 #471

Merged
merged 1 commit into from
Nov 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ This target is tested with all actively supported [Python](https://devguide.pyth
| password | False | None | Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. |
| database | False | None | Database name. Note if sqlalchemy_url is set this will be ignored. |
| sqlalchemy_url | False | None | SQLAlchemy connection string. This will override using host, user, password, port, dialect, and all ssl settings. Note that you must escape password special characters properly. See https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords |
| dialect+driver | False | postgresql+psycopg2 | Dialect+driver see https://docs.sqlalchemy.org/en/20/core/engines.html. Generally just leave this alone. Note if sqlalchemy_url is set this will be ignored. |
| dialect+driver | False | postgresql+psycopg | Dialect+driver see https://docs.sqlalchemy.org/en/20/core/engines.html. Generally just leave this alone. Note if sqlalchemy_url is set this will be ignored. |
| default_target_schema | False | melty | Postgres schema to send data to, example: tap-clickup |
| activate_version | False | 1 | If set to false, the tap will ignore activate version messages. If set to true, add_record_metadata must be set to true as well. |
| hard_delete | False | 0 | When activate version is sent from a tap this specefies if we should delete the records that don't match, or mark them with a date in the `_sdc_deleted_at` column. This config option is ignored if `activate_version` is set to false. |
Expand Down
230 changes: 168 additions & 62 deletions poetry.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.9"
faker = {version = "~=30.0", optional = true}
psycopg = {extras = ["binary"], version = "3.2.3"}
psycopg2-binary = "2.9.10"
sqlalchemy = "~=2.0"
sshtunnel = "0.4.0"
Expand Down
2 changes: 1 addition & 1 deletion target_postgres/target.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ def __init__(
th.Property(
"dialect+driver",
th.StringType,
default="postgresql+psycopg2",
default="postgresql+psycopg",
description=(
"Dialect+driver see "
+ "https://docs.sqlalchemy.org/en/20/core/engines.html. "
Expand Down
4 changes: 2 additions & 2 deletions target_postgres/tests/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

def postgres_config():
return {
"dialect+driver": "postgresql+psycopg2",
"dialect+driver": "postgresql+psycopg",
"host": "localhost",
"user": "postgres",
"password": "postgres",
Expand All @@ -29,7 +29,7 @@ def postgres_config():

def postgres_config_no_ssl():
return {
"dialect+driver": "postgresql+psycopg2",
"dialect+driver": "postgresql+psycopg",
"host": "localhost",
"user": "postgres",
"password": "postgres",
Expand Down
4 changes: 3 additions & 1 deletion target_postgres/tests/test_sdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,9 @@ class BasePostgresSDKTests:
@pytest.fixture()
def connection(self, runner):
engine = create_engine(runner)
return engine.connect()
with engine.connect() as conn:
yield conn
engine.dispose()


SDKTests = get_target_test_class(
Expand Down
12 changes: 10 additions & 2 deletions target_postgres/tests/test_target_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ def verify_data(
sqlalchemy.text(f"SELECT COUNT(*) FROM {full_table_name}")
)
assert result.first()[0] == number_of_rows
engine.dispose()


def test_sqlalchemy_url_config(postgres_config_no_ssl):
Expand All @@ -167,7 +168,7 @@ def test_sqlalchemy_url_config(postgres_config_no_ssl):
def test_port_default_config():
"""Test that the default config is passed into the engine when the config doesn't provide it"""
config = {
"dialect+driver": "postgresql+psycopg2",
"dialect+driver": "postgresql+psycopg",
"host": "localhost",
"user": "postgres",
"password": "postgres",
Expand All @@ -186,12 +187,13 @@ def test_port_default_config():
engine.url.render_as_string(hide_password=False)
== f"{dialect_driver}://{user}:{password}@{host}:5432/{database}"
)
engine.dispose()


def test_port_config():
"""Test that the port config works"""
config = {
"dialect+driver": "postgresql+psycopg2",
"dialect+driver": "postgresql+psycopg",
"host": "localhost",
"user": "postgres",
"password": "postgres",
Expand All @@ -211,6 +213,7 @@ def test_port_config():
engine.url.render_as_string(hide_password=False)
== f"{dialect_driver}://{user}:{password}@{host}:5433/{database}"
)
engine.dispose()


# Test name would work well
Expand Down Expand Up @@ -402,6 +405,7 @@ def test_no_primary_keys(postgres_target):
singer_file_to_target(file_name, postgres_target)

verify_data(postgres_target, table_name, 16)
engine.dispose()


def test_no_type(postgres_target):
Expand Down Expand Up @@ -511,6 +515,7 @@ def test_anyof(postgres_target):
# {"anyOf":[{"type":"string"},{"type":"integer"},{"type":"null"}]}
if column.name == "legacy_id":
assert isinstance(column.type, TEXT)
engine.dispose()


def test_new_array_column(postgres_target):
Expand Down Expand Up @@ -621,6 +626,7 @@ def test_activate_version_hard_delete(postgres_config_no_ssl):
assert result.rowcount == 9

singer_file_to_target(file_name, pg_hard_delete_true)
engine.dispose()

# Should remove the 2 records we added manually
with engine.connect() as connection:
Expand Down Expand Up @@ -692,6 +698,7 @@ def test_activate_version_soft_delete(postgres_config_no_ssl):
# South America row should not have been modified, but it would have been prior
# to the fix mentioned in #204 and implemented in #240.
assert south_america == result.first()._asdict()
engine.dispose()


def test_activate_version_no_metadata(postgres_config_no_ssl):
Expand Down Expand Up @@ -742,6 +749,7 @@ def test_activate_version_deletes_data_properly(postgres_target):
with engine.connect() as connection:
result = connection.execute(sqlalchemy.text(f"SELECT * FROM {full_table_name}"))
assert result.rowcount == 0
engine.dispose()


def test_reserved_keywords(postgres_target):
Expand Down
2 changes: 1 addition & 1 deletion target_postgres/tests/test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def connector():
"""Create a PostgresConnector instance."""
return PostgresConnector(
config={
"dialect+driver": "postgresql+psycopg2",
"dialect+driver": "postgresql+psycopg",
"host": "localhost",
"port": "5432",
"user": "postgres",
Expand Down