diff --git a/Makefile b/Makefile index f0b3855c..1904d6e1 100644 --- a/Makefile +++ b/Makefile @@ -60,9 +60,5 @@ dev-install: build .PHONY: dev-install dev-sandbox: - ( cd sandbox \ - && mkdir -p volumes \ - && cp -r fixtures/* volumes/ \ - && docker-compose up \ - ; docker-compose down ) + ( cd sandbox && docker-compose up --build ; docker-compose down ) .PHONY: dev-sandbox diff --git a/sandbox/.env b/sandbox/.env index 38108d1e..9bea6552 100644 --- a/sandbox/.env +++ b/sandbox/.env @@ -3,4 +3,7 @@ POSTGRES_PASSWORD=dbtmetabase1 POSTGRES_DB=dbtmetabase POSTGRES_SCHEMA=public POSTGRES_PORT=5432 +MB_SETUP_TOKEN=2ffe1bc9-4bc5-4184-a10d-731517d8e4a3 +MB_USER=dbtmetabase@example.com +MB_PASSWORD=dbtmetabase1 MB_PORT=3000 diff --git a/sandbox/Dockerfile b/sandbox/Dockerfile index c1913355..be77e08d 100644 --- a/sandbox/Dockerfile +++ b/sandbox/Dockerfile @@ -1,11 +1,14 @@ FROM python:3.11-slim-bullseye -RUN python3 -m pip install dbt-postgres - WORKDIR /app +COPY requirements.txt ./ + +RUN python3 -m pip install -r requirements.txt + +COPY entrypoint.py ./ COPY dbt_project.yml ./ COPY models/ ./models/ COPY profiles.yml ./ -CMD dbt run --profiles-dir . +ENTRYPOINT ["python3", "entrypoint.py"] diff --git a/sandbox/config.yml b/sandbox/config.yml deleted file mode 100644 index 611cea97..00000000 --- a/sandbox/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -config: - dbt_manifest_path: ./sandbox/target/manifest.json - dbt_database: dbtmetabase - metabase_url: http://localhost:3000 - # Pre-configured in Metabase fixtures - metabase_database: dbtmetabase - metabase_username: dbtmetabase@example.com - metabase_password: dbtmetabase1 diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index 36e4afdc..57067959 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -13,32 +13,36 @@ services: - ${POSTGRES_PORT}:5432 networks: - common - volumes: - - postgres-data:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U $POSTGRES_USER"] interval: 5s timeout: 5s retries: 5 + start_period: 10s restart: always metabase: image: metabase/metabase:latest environment: - - MB_DB_FILE=/metabase.db + - MB_SETUP_TOKEN=${MB_SETUP_TOKEN:-} ports: - ${MB_PORT}:3000 networks: - common - volumes: - - ./volumes/metabase.db:/metabase.db + healthcheck: + test: ["CMD-SHELL", "curl --fail http://localhost:3000/api/health"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s restart: always depends_on: postgres: condition: service_healthy - dbt: + app: build: . + command: init environment: - POSTGRES_USER=${POSTGRES_USER:-} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-} @@ -46,6 +50,11 @@ services: - POSTGRES_SCHEMA=${POSTGRES_SCHEMA:-} - POSTGRES_PORT=${POSTGRES_PORT:-} - POSTGRES_HOST=postgres + - MB_SETUP_TOKEN=${MB_SETUP_TOKEN:-} + - MB_USER=${MB_USER:-} + - MB_PASSWORD=${MB_PASSWORD:-} + - MB_PORT=${MB_PORT:-} + - MB_HOST=metabase networks: - common volumes: @@ -53,9 +62,8 @@ services: depends_on: postgres: condition: service_healthy + metabase: + condition: service_healthy networks: common: - -volumes: - postgres-data: diff --git a/sandbox/entrypoint.py b/sandbox/entrypoint.py new file mode 100755 index 00000000..a5626bfd --- /dev/null +++ b/sandbox/entrypoint.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import requests +from molot import target, envarg, envarg_int, evaluate, shell + +POSTGRES_HOST = envarg("POSTGRES_HOST") +POSTGRES_PORT = envarg_int("POSTGRES_PORT") +POSTGRES_DB = envarg("POSTGRES_DB") +POSTGRES_USER = envarg("POSTGRES_USER") +POSTGRES_PASSWORD = envarg("POSTGRES_PASSWORD") +MB_HOST = envarg("MB_HOST") +MB_PORT = envarg_int("MB_PORT") +MB_SETUP_TOKEN = envarg("MB_SETUP_TOKEN") +MB_USER = envarg("MB_USER") +MB_PASSWORD = envarg("MB_PASSWORD") +MB_NAME = envarg("MB_NAME", "dbtmetabase") + + +@target(description="set up Metabase user and database") +def setup_metabase(): + requests.post( + f"http://{MB_HOST}:{MB_PORT}/api/setup", + json={ + "token": MB_SETUP_TOKEN, + "user": { + "site_name": MB_NAME, + "first_name": MB_NAME, + "last_name": None, + "email": MB_USER, + "password_confirm": MB_PASSWORD, + "password": MB_PASSWORD, + }, + "database": { + "engine": "postgres", + "name": POSTGRES_DB, + "details": { + "host": POSTGRES_HOST, + "port": POSTGRES_PORT, + "dbname": POSTGRES_DB, + "user": POSTGRES_USER, + "password": POSTGRES_PASSWORD, + "schema-filters-type": "all", + "ssl": False, + "tunnel-enabled": False, + "advanced-options": False, + }, + "is_on_demand": False, + "is_full_sync": True, + "is_sample": False, + "cache_ttl": None, + "refingerprint": False, + "auto_run_queries": True, + "schedules": {}, + }, + "prefs": { + "site_name": MB_NAME, + "site_locale": "en", + "allow_tracking": "false", + }, + }, + timeout=10, + ).raise_for_status() + + +@target(description="run dbt project") +def run_dbt(): + shell("dbt run --profiles-dir .") + + +@target(description="initial setup", depends=["setup_metabase", "run_dbt"]) +def init(): + pass + + +evaluate() diff --git a/sandbox/fixtures/metabase.db/metabase.db.mv.db b/sandbox/fixtures/metabase.db/metabase.db.mv.db deleted file mode 100644 index 9a0b5674..00000000 Binary files a/sandbox/fixtures/metabase.db/metabase.db.mv.db and /dev/null differ diff --git a/sandbox/requirements.txt b/sandbox/requirements.txt new file mode 100644 index 00000000..281f4f31 --- /dev/null +++ b/sandbox/requirements.txt @@ -0,0 +1,3 @@ +molot +requests +dbt-postgres