From 6f3d9eb16c98412241886ccff4cd0ddc7c95d198 Mon Sep 17 00:00:00 2001 From: Mike Gouline <1960272+gouline@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:03:46 +1000 Subject: [PATCH 1/4] Linux environment fixes --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 068fca0b..8f77876c 100644 --- a/Makefile +++ b/Makefile @@ -63,15 +63,15 @@ install: build .PHONY: install sandbox-up: - ( cd sandbox && docker-compose up --build --attach app ) + ( cd sandbox && docker compose up --build --attach app ) .PHONY: sandbox-up sandbox-down: - ( cd sandbox && docker-compose down ) + ( cd sandbox && docker compose down ) .PHONY: sandbox-up sandbox-models: - ( source sandbox/.env && python3 -m dbtmetabase models \ + ( . sandbox/.env && python3 -m dbtmetabase models \ --manifest-path sandbox/target/manifest.json \ --metabase-url http://localhost:$$MB_PORT \ --metabase-username $$MB_USER \ @@ -86,7 +86,7 @@ sandbox-models: sandbox-exposures: rm -rf sandbox/models/exposures mkdir -p sandbox/models/exposures - ( source sandbox/.env && python3 -m dbtmetabase exposures \ + ( . sandbox/.env && python3 -m dbtmetabase exposures \ --manifest-path sandbox/target/manifest.json \ --metabase-url http://localhost:$$MB_PORT \ --metabase-username $$MB_USER \ @@ -95,7 +95,7 @@ sandbox-exposures: --output-grouping collection \ --verbose ) - ( source sandbox/.env && cd sandbox && \ + ( . sandbox/.env && cd sandbox && \ POSTGRES_HOST=localhost \ POSTGRES_PORT=$$POSTGRES_PORT \ POSTGRES_USER=$$POSTGRES_USER \ From b2286c01f9915b4ed172534b68ed6db2ae70f2df Mon Sep 17 00:00:00 2001 From: Mike Gouline <1960272+gouline@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:04:14 +1000 Subject: [PATCH 2/4] Pin dependencies for reproducibility --- requirements-test.txt | 4 ++-- sandbox/docker-compose.yml | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index c35ccd81..0a7f88cc 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -5,7 +5,7 @@ black>=23.11.0 isort>=5.12.0 pylint>=3.0.2 mypy>=1.7.1 -molot>=1.0.0,<1.1.0 -dbt-postgres>=1.7.4,<2.0.0 +molot~=1.0.0 +dbt-postgres~=1.7.16 types-requests types-PyYAML diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index a7063e7d..43b28862 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - name: dbt-metabase-sandbox services: @@ -22,7 +20,7 @@ services: restart: always metabase: - image: metabase/metabase:latest + image: metabase/metabase:v0.50.0 environment: - MB_SETUP_TOKEN=${MB_SETUP_TOKEN:-} ports: From 2dfbab591f229fee1e533f8a2d8e559dc682a94e Mon Sep 17 00:00:00 2001 From: Mike Gouline <1960272+gouline@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:04:26 +1000 Subject: [PATCH 3/4] Metabase 50 compatibility fixes --- dbtmetabase/metabase.py | 4 ++++ sandbox/entrypoint.py | 46 ++++++++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/dbtmetabase/metabase.py b/dbtmetabase/metabase.py index 8410d93c..93493c85 100644 --- a/dbtmetabase/metabase.py +++ b/dbtmetabase/metabase.py @@ -185,6 +185,10 @@ def find_user(self, uid: str) -> Optional[Mapping]: if error.response.status_code == 404: _logger.warning("User '%s' not found", uid) return None + elif error.response.status_code == 400 and "internal user" in error.response.text: + # Since X.50.0 fetching internal user raises 400 Not able to modify the internal user + _logger.warning("User '%s' is internal", uid) + return None raise def update_table(self, uid: str, body: Mapping) -> Mapping: diff --git a/sandbox/entrypoint.py b/sandbox/entrypoint.py index 96e3cf84..4ff27b1b 100755 --- a/sandbox/entrypoint.py +++ b/sandbox/entrypoint.py @@ -31,6 +31,16 @@ def dbt_run(): shell("dbt run --profiles-dir .") +def _session_headers(): + session_id = requests.post( + url=f"http://{MB_HOST}:{MB_PORT}/api/session", + json={"username": MB_USER, "password": MB_PASSWORD}, + timeout=10, + ).json()["id"] + + return {"X-Metabase-Session": session_id} + + @target(description="set up Metabase user and database") def metabase_setup(): requests.post( @@ -76,16 +86,38 @@ def metabase_setup(): timeout=10, ).raise_for_status() + requests.post( + url=f"http://{MB_HOST}:{MB_PORT}/api/database", + headers=_session_headers(), + json={ + "engine": "postgres", + "name": POSTGRES_DB, + "details": { + "host": POSTGRES_HOST, + "port": POSTGRES_PORT, + "dbname": POSTGRES_DB, + "user": POSTGRES_USER, + "password": POSTGRES_PASSWORD, + "schema-filters-type": "all", + "ssl": False, + "tunnel-enabled": False, + "advanced-options": False, + }, + "is_on_demand": False, + "is_full_sync": True, + "is_sample": False, + "cache_ttl": None, + "refingerprint": False, + "auto_run_queries": True, + "schedules": {}, + }, + timeout=10, + ).raise_for_status() + @target(description="add mock content to Metabase") def metabase_content(): - session_id = requests.post( - url=f"http://{MB_HOST}:{MB_PORT}/api/session", - json={"username": MB_USER, "password": MB_PASSWORD}, - timeout=10, - ).json()["id"] - - headers = {"X-Metabase-Session": session_id} + headers = _session_headers() database_id = "" databases = requests.get( From 42f8a544f8a67a2bd1973b2df6df4b9eae03e2bb Mon Sep 17 00:00:00 2001 From: Mike Gouline <1960272+gouline@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:17:15 +1000 Subject: [PATCH 4/4] Fmt --- dbtmetabase/metabase.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbtmetabase/metabase.py b/dbtmetabase/metabase.py index 93493c85..c50917bd 100644 --- a/dbtmetabase/metabase.py +++ b/dbtmetabase/metabase.py @@ -185,7 +185,10 @@ def find_user(self, uid: str) -> Optional[Mapping]: if error.response.status_code == 404: _logger.warning("User '%s' not found", uid) return None - elif error.response.status_code == 400 and "internal user" in error.response.text: + elif ( + error.response.status_code == 400 + and "internal user" in error.response.text + ): # Since X.50.0 fetching internal user raises 400 Not able to modify the internal user _logger.warning("User '%s' is internal", uid) return None