Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Initial implementation of HTTP connector #1649

Open
wants to merge 30 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
002c4f8
Initial implementation of HTTP connector
edgarrmondragon Apr 28, 2023
9ea4189
Merge branch 'main' into http-connector
May 24, 2023
86ce0a1
Merge branch 'main' into http-connector
edgarrmondragon May 24, 2023
375bfa3
Merge branch 'main' into http-connector
edgarrmondragon Jul 11, 2023
a408431
Feedback: Address private method and attribute access for connector API
edgarrmondragon Jul 11, 2023
79c4f56
Fix warning line
edgarrmondragon Jul 11, 2023
9045268
Test deprecations
edgarrmondragon Jul 12, 2023
536bb2f
Fix session type annotation
edgarrmondragon Jul 12, 2023
8927a0e
Merge branch 'main' into http-connector
edgarrmondragon Jul 12, 2023
f0619cf
Merge branch 'main' into http-connector
edgarrmondragon Jul 14, 2023
e72e193
Merge branch 'main' into http-connector
edgarrmondragon Jul 14, 2023
f2c442a
Merge branch 'main' into http-connector
edgarrmondragon Jul 17, 2023
557b121
Merge branch 'main' into http-connector
edgarrmondragon Jul 18, 2023
fe06ab2
Merge branch 'main' into http-connector
edgarrmondragon Jul 18, 2023
493ba22
Merge branch 'main' into http-connector
edgarrmondragon Jul 27, 2023
138efbe
Rename type var
edgarrmondragon Jul 27, 2023
7fec03e
Fix types
edgarrmondragon Jul 27, 2023
1b5b753
Merge branch 'main' into http-connector
edgarrmondragon Jul 27, 2023
4f3f2d5
Merge branch 'main' into http-connector
edgarrmondragon Aug 2, 2023
dcf83f3
Merge branch 'main' into http-connector
edgarrmondragon Dec 7, 2023
5a53ce7
Merge branch 'main' into http-connector
edgarrmondragon Jan 11, 2024
e5a7c99
Merge branch 'main' into http-connector
edgarrmondragon Jan 19, 2024
ecb522a
Merge branch 'main' into http-connector
edgarrmondragon Jan 22, 2024
ec574c8
Merge branch 'main' into http-connector
edgarrmondragon Jan 24, 2024
87f94a5
Merge branch 'main' into http-connector
edgarrmondragon Jan 30, 2024
495d744
Merge branch 'main' into http-connector
edgarrmondragon Jan 31, 2024
1c3b6f0
Merge branch 'main' into http-connector
edgarrmondragon May 29, 2024
0b35bde
chore: Run `poetry lock` to install the latest transitive dependencies
edgarrmondragon May 29, 2024
eedc63f
Make Ruff happy
edgarrmondragon May 29, 2024
911da30
Merge branch 'main' into http-connector
edgarrmondragon Aug 9, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'main' into http-connector
edgarrmondragon committed Dec 7, 2023
commit dcf83f3c5ae7326054cae133fd722fb28ceb5bce
3 changes: 1 addition & 2 deletions noxfile.py
Original file line number Diff line number Diff line change
@@ -43,11 +43,10 @@
"duckdb",
"duckdb-engine",
"pytest",
"pytest-benchmark",
"pytest-durations",
"pytest-httpserver",
"pytest-snapshot",
"freezegun",
"pandas",
"pyarrow",
"requests-mock",
"time-machine",
101 changes: 3 additions & 98 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 24 additions & 11 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -120,17 +120,30 @@ coverage = [
{extras = ["toml"], version = ">=7.2,<7.3", python = "<3.8"},
{extras = ["toml"], version = ">=7.2", python = ">=3.8"},
]
requests-mock = "^1.10.0"
types-jsonschema = "^4.17.0.6"
types-python-dateutil = "^2.8.19"
types-pytz = ">=2022.7.1.2,<2024.0.0.0"
types-requests = "^2.28.11"
types-simplejson = "^3.18.0"
types-PyYAML = "^6.0.12"
coverage = {extras = ["toml"], version = "^7.2"}
pyarrow = ">=11,<13"
pytest-httpserver = "^1.0.6"
pytest-snapshot = "^0.9.0"
duckdb = ">=0.8.0"
duckdb-engine = ">=0.9.2"
mypy = [
{ version = ">=1.0,<1.5", python = "<3.8" },
{ version = ">=1.0", python = ">=3.8" },
]
pytest-benchmark = ">=4.0.0"
pytest-httpserver = { version = ">=1.0.6", python = "<4" }
pytest-snapshot = ">=0.9.0"
requests-mock = ">=1.10.0"
time-machine = [
{ version = ">=2.10.0,<2.11", python = "<3.8" },
{ version = ">=2.10.0", python = ">=3.8" },
]
types-jsonschema = [
{ version = ">=4.17.0.6,<4.18", python = "<3.8" },
{ version = ">=4.17.0.6", python = ">=3.8" },
]
types-python-dateutil = ">=2.8.19"
types-pytz = ">=2022.7.1.2"
types-requests = ">=2.28.11"
types-simplejson = ">=3.18.0"
types-PyYAML = ">=6.0.12"
xdoctest = ">=1.1.1"

[tool.poetry.group.benchmark.dependencies]
pytest-codspeed = ">=2.2.0"
6 changes: 3 additions & 3 deletions singer_sdk/connectors/sql.py
Original file line number Diff line number Diff line change
@@ -686,7 +686,7 @@ def create_schema(self, schema_name: str) -> None:
Args:
schema_name: The target schema to create.
"""
with self.connect() as conn:
with self.connect() as conn, conn.begin():
conn.execute(sqlalchemy.schema.CreateSchema(schema_name))

def create_empty_table(
@@ -867,7 +867,7 @@ def rename_column(self, full_table_name: str, old_name: str, new_name: str) -> N
column_name=old_name,
new_column_name=new_name,
)
with self.connect() as conn:
with self.connect() as conn, conn.begin():
conn.execute(column_rename_ddl)

def merge_sql_types(
@@ -1173,7 +1173,7 @@ def _adapt_column_type(
column_name=column_name,
column_type=compatible_sql_type,
)
with self.connect() as conn:
with self.connect() as conn, conn.begin():
conn.execute(alter_column_ddl)

def serialize_json(self, obj: object) -> str:
20 changes: 19 additions & 1 deletion singer_sdk/helpers/_compat.py
Original file line number Diff line number Diff line change
@@ -22,4 +22,22 @@
else:
from importlib import resources

__all__ = ["metadata", "final", "resources", "Protocol"]
if sys.version_info < (3, 11):
from backports.datetime_fromisoformat import MonkeyPatch

MonkeyPatch.patch_fromisoformat()

datetime_fromisoformat = datetime.datetime.fromisoformat
date_fromisoformat = datetime.date.fromisoformat
time_fromisoformat = datetime.time.fromisoformat

__all__ = [
"metadata",
"final",
"resources",
"entry_points",
"datetime_fromisoformat",
"date_fromisoformat",
"time_fromisoformat",
"Protocol",
]
7 changes: 4 additions & 3 deletions tests/core/test_streams.py
Original file line number Diff line number Diff line change
@@ -526,9 +526,10 @@ def discover_streams(self):
return [SelectedStream(self), UnselectedStream(self)]

# Check that the selected stream is selected
tap = MyTap(config=None, catalog=input_catalog)
for stream in selection:
assert tap.streams[stream].selected is selection[stream]
tap = MyTap(config=None, catalog=input_catalog, validate_config=False)
assert all(
tap.streams[stream].selected is selection[stream] for stream in selection
)


def test_deprecations(tap: SimpleTestTap):
You are viewing a condensed version of this merge commit. You can view the full changes here.