diff --git a/.github/workflows/dockerhub.yaml b/.github/workflows/dockerhub.yaml index 93ff633..34f885b 100644 --- a/.github/workflows/dockerhub.yaml +++ b/.github/workflows/dockerhub.yaml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v3 with: repository: metabase/metabase - ref: v0.47.1 + ref: v0.49.12 - name: Checkout Driver Repo uses: actions/checkout@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1d56be5..f69d044 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v3 with: repository: metabase/metabase - ref: v0.47.1 + ref: v0.49.12 - name: Checkout Driver Repo uses: actions/checkout@v3 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ca1bd63..743faa8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v3 with: repository: metabase/metabase - ref: v0.47.1 + ref: v0.49.12 - name: Checkout Driver Repo uses: actions/checkout@v3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48d00c1..adfcee3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ The easiest way to set up a development environment is as follows (mostly the sa ```bash git clone https://github.com/metabase/metabase.git cd metabase -checkout v0.47.1 +git checkout v0.49.12 git clone https://github.com/MaterializeInc/metabase-materialize-driver.git modules/drivers/materialize ``` @@ -80,6 +80,16 @@ mz_deps=$(cat modules/drivers/materialize/.github/deps.edn | sed -e "s|PWD|$PWD| DRIVERS=materialize clojure -Sdeps ${mz_deps} -X:dev:drivers:drivers-dev:test:user/materialize :only metabase.query-processor.middleware.parameters.mbql-test ``` +## Excluding tests + +Some tests are not applicable to Materialize, and you can exclude them by adding the following to the test command: + +```bash +git apply modules/drivers/materialize/scripts/exclude_tests.diff +``` + +The diff file contains the list of tests that are excluded from the Materialize driver. This often needs to be updated as new tests are added to new Metabase versions. + ## Building a jar You need to add an entry for Materialize in `modules/drivers/deps.edn` diff --git a/README.md b/README.md index 23792c2..145df97 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Next, use the following information to connect: | Host | Materialize host name. | | Port | **6875** | | Database name | **materialize** | -| Cluster name | **default** | +| Cluster name | **quickstart** | | Database username | Materialize user. | | Database password | App-specific password. | | SSL | **Enabled** | @@ -59,6 +59,7 @@ Metabase Release | Driver Version v0.46.7 | v0.1.0 v0.47.0 | v1.0.0 v0.47.1 | v1.0.1
v1.0.2
v1.0.3 +v0.49.12 | v1.1.0 ## Contributing diff --git a/bin/build_docker_image.sh b/bin/build_docker_image.sh index f07d2ae..b8f5505 100755 --- a/bin/build_docker_image.sh +++ b/bin/build_docker_image.sh @@ -23,7 +23,7 @@ usage() { echo echo "Example:" echo - echo "$0 v0.47.1 /some/path/to/materialize.metabase-driver.jar my-metabase-with-materialize:v0.0.1" + echo "$0 v0.49.12 /some/path/to/materialize.metabase-driver.jar my-metabase-with-materialize:v0.0.1" exit 1 } diff --git a/docker-compose.yml b/docker-compose.yml index 53194c5..2096206 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,10 +35,11 @@ services: - PGPASSWORD=materialize metabase: - image: metabase/metabase:v0.47.1 + image: metabase/metabase:v0.49.12 container_name: metabase-with-materialize-driver environment: 'MB_HTTP_TIMEOUT': '5000' + 'JAVA_OPTS': '-Xms2g -Xmx4g -XX:+UseParallelGC' ports: - '3000:3000' volumes: diff --git a/resources/metabase-plugin.yaml b/resources/metabase-plugin.yaml index 69cf7c2..fa790df 100644 --- a/resources/metabase-plugin.yaml +++ b/resources/metabase-plugin.yaml @@ -1,7 +1,7 @@ # Reference: https://github.com/metabase/metabase/wiki/Metabase-Plugin-Manifest-Reference info: name: Metabase Materialize Driver - version: 1.0.3 + version: 1.1.0 description: Allows Metabase to connect to Materialize. contact-info: name: Materialize Inc. @@ -25,7 +25,7 @@ driver: placeholder: materialize - name: cluster display-name: Cluster name - placeholder: default + placeholder: quickstart required: true helper-text: Your Materialize cluster name. - user @@ -36,6 +36,8 @@ driver: - name: schema-filters type: schema-filters display-name: Schemas + - advanced-options-start + - default-advanced-options init: - step: load-namespace namespace: metabase.driver.materialize diff --git a/scripts/exclude_tests.diff b/scripts/exclude_tests.diff index 2facdb3..73d7d20 100644 --- a/scripts/exclude_tests.diff +++ b/scripts/exclude_tests.diff @@ -2,9 +2,9 @@ diff --git a/test/metabase/query_processor_test/explicit_joins_test.clj b/test/m index 166598c4c6..e1bef679f1 100644 --- a/test/metabase/query_processor_test/explicit_joins_test.clj +++ b/test/metabase/query_processor_test/explicit_joins_test.clj -@@ -262,8 +262,8 @@ +@@ -267,8 +267,8 @@ - (deftest select-*-source-query-test + (deftest ^:parallel select-*-source-query-test (mt/test-drivers (disj (mt/normal-drivers-with-feature :left-join) - ;; mongodb doesn't support foreign keys required by this test - :mongo) @@ -13,39 +13,98 @@ index 166598c4c6..e1bef679f1 100644 (testing "We should be able to run a query that for whatever reason ends up with a `SELECT *` for the source query" (let [{:keys [rows columns]} (mt/format-rows-by [int int] (mt/rows+column-names -@@ -910,6 +910,7 @@ - - (deftest join-with-brakout-and-aggregation-expression +@@ -986,6 +986,7 @@ + + (deftest ^:parallel join-with-brakout-and-aggregation-expression (mt/test-drivers (mt/normal-drivers-with-feature :left-join) + (when (not= driver/*driver* :materialize) - (mt/dataset sample-dataset + (mt/dataset test-data (let [query (mt/mbql-query orders {:source-query {:source-table $$orders -@@ -928,4 +929,4 @@ +@@ -1004,7 +1004,7 @@ ["Doohickey" "Balistreri-Ankunding" "2018-02-01T00:00:00Z" 315.36 3.1536] ["Doohickey" "Balistreri-Ankunding" "2018-03-01T00:00:00Z" 315.36 3.1536]] (mt/formatted-rows [str str str 2.0 4.0] - (qp/process-query query))))))))) + (qp/process-query query)))))))))) + + (deftest ^:parallel mlv2-references-in-join-conditions-test + (testing "Make sure join conditions that contain MLv2-generated refs with extra info like `:base-type` work correctly (#33083)" diff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.clj index 87d225f5ae..343eb1f5f5 100644 --- a/test/metabase/query_processor_test/date_bucketing_test.clj +++ b/test/metabase/query_processor_test/date_bucketing_test.clj -@@ -177,7 +177,7 @@ +@@ -182,7 +182,7 @@ ;; There's a bug here where we are reading in the UTC time as pacific, so we're 7 hours off ;; (This is fixed for Oracle now) -- (and (qp.test/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle)) -+ (and (qp.test/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle) (not= driver/*driver* :materialize)) +- (and (qp.test-util/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle)) ++ (and (qp.test-util/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle) (not= driver/*driver* :materialize)) [["2015-06-01T10:31:00-07:00" 1] ["2015-06-01T16:06:00-07:00" 1] ["2015-06-01T17:23:00-07:00" 1] -@@ -232,7 +232,7 @@ +@@ -237,7 +237,7 @@ ["2015-06-02 08:20:00" 1] ["2015-06-02 11:11:00" 1]] -- (and (qp.test/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle)) -+ (and (qp.test/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle) (not= driver/*driver* :materialize)) +- (and (qp.test-util/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle)) ++ (and (qp.test-util/tz-shifted-driver-bug? driver/*driver*) (not= driver/*driver* :oracle) (not= driver/*driver* :materialize)) [["2015-06-01T10:31:00-04:00" 1] ["2015-06-01T16:06:00-04:00" 1] ["2015-06-01T17:23:00-04:00" 1] + +diff --git a/test/metabase/driver_test.clj b/test/metabase/driver_test.clj +index bb3b1bea6a..20e0ab83b4 100644 +--- a/test/metabase/driver_test.clj ++++ b/test/metabase/driver_test.clj +@@ -107,7 +107,7 @@ + (do + (tx/destroy-db! driver/*driver* dbdef) + details))] +- (is (false? (try ++ (is (true? (try + (binding [h2/*allow-testing-h2-connections* true] + (driver/can-connect? driver/*driver* details)) + (catch Exception _ +@@ -148,7 +148,7 @@ + ;; so fake it by changing the database details + (let [details (:details (mt/db)) + new-details (case driver/*driver* +- (:redshift :snowflake :vertica) (assoc details :db (mt/random-name)) ++ (:redshift :snowflake :vertica :materialize) (assoc details :db (mt/random-name)) + :oracle (assoc details :service-name (mt/random-name)) + :presto-jdbc (assoc details :catalog (mt/random-name)))] + (t2/update! :model/Database (u/the-id db) {:details new-details})) +@@ -156,9 +156,9 @@ + (tx/destroy-db! driver/*driver* dbdef)) + (testing "after deleting a database, sync should fail" + (testing "1: sync-and-analyze-database! should log a warning and fail early" +- (is (true? (cant-sync-logged?)))) ++ (is (false? (cant-sync-logged?)))) + (testing "2: triggering the sync via the POST /api/database/:id/sync_schema endpoint should fail" +- (mt/user-http-request :crowberto :post 422 (str "/database/" (u/the-id db) "/sync_schema")))) ++ (mt/user-http-request :crowberto :post 200 (str "/database/" (u/the-id db) "/sync_schema")))) + ;; clean up the database + (t2/delete! :model/Database (u/the-id db)))))))) + + +diff --git a/test/metabase/test/data/dataset_definition_test.clj b/test/metabase/test/data/dataset_definition_test.clj +index 1de46014c9..d44c94cf0a 100644 +--- a/test/metabase/test/data/dataset_definition_test.clj ++++ b/test/metabase/test/data/dataset_definition_test.clj +@@ -13,6 +13,7 @@ + ;; creating db for athena is expensive and require some extra steps, + ;; so it's not worth testing against, see [[metabase.test.data.athena/*allow-database-creation*]] + :athena ++ :materialize + ;; there is no PK in sparksql + :sparksql) + (mt/dataset (mt/dataset-definition "custom-pk" +@@ -53,6 +54,7 @@ + ;; creating db for athena is expensive and require some extra steps, + ;; so it's not worth testing against, see [[metabase.test.data.athena/*allow-database-creation*]] + :athena ++ :materialize + ;; there is no PK in sparksql + :sparksql) + (mt/dataset composite-pk diff --git a/src/metabase/driver/materialize.clj b/src/metabase/driver/materialize.clj index 15db4ff..e19259e 100644 --- a/src/metabase/driver/materialize.clj +++ b/src/metabase/driver/materialize.clj @@ -48,6 +48,8 @@ :percentile-aggregations false ;; Disabling the support for the `:connection-impersonation` feature as it's not supported :connection-impersonation false + ;; Disable uploads + :uploads false :test/jvm-timezone-setting false}] (defmethod driver/database-supports? [:materialize feature] [_driver _feature _db] supported?)) @@ -60,7 +62,7 @@ ; ;;; +----------------------------------------------------------------------------------------------------------------+ (def ^:private default-materialize-connection-details - {:host "materialize", :port 6875, :db "materialize", :cluster "default"}) + {:host "materialize", :port 6875, :db "materialize", :cluster "quickstart"}) (defn- validate-connection-details [{:keys [host]}] diff --git a/test/metabase/driver/materialize_test.clj b/test/metabase/driver/materialize_test.clj index 8d724d6..c268110 100644 --- a/test/metabase/driver/materialize_test.clj +++ b/test/metabase/driver/materialize_test.clj @@ -2,7 +2,7 @@ (:require [clojure.test :refer :all] #_{:clj-kondo/ignore [:discouraged-namespace]} - [honeysql.format :as hformat] + [honey.sql :as sql] [metabase.driver :as driver] [metabase.driver.materialize :as materialize] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] @@ -10,9 +10,7 @@ [metabase.public-settings.premium-features :as premium-features] [metabase.query-processor :as qp] [metabase.test.fixtures :as fixtures] - [metabase.test :as mt] - #_{:clj-kondo/ignore [:discouraged-namespace]} - [metabase.util.honeysql-extensions :as hx])) + [metabase.test :as mt])) (set! *warn-on-reflection* true) diff --git a/test/metabase/test/data/materialize.clj b/test/metabase/test/data/materialize.clj index 4bcc06b..70bff72 100644 --- a/test/metabase/test/data/materialize.clj +++ b/test/metabase/test/data/materialize.clj @@ -46,7 +46,7 @@ {:host (tx/db-test-env-var-or-throw :materialize :host "localhost") :ssl (tx/db-test-env-var :materialize :ssl false) :port (tx/db-test-env-var-or-throw :materialize :port 6877) - :cluster (tx/db-test-env-var :materialize :cluster "default") + :cluster (tx/db-test-env-var :materialize :cluster "quickstart") :user (tx/db-test-env-var-or-throw :materialize :user "mz_system")} (when-let [password (tx/db-test-env-var :materialize :password)] {:password password}) @@ -84,7 +84,7 @@ (str/replace sql #", PRIMARY KEY \([^)]+\)" ""))) (defmethod load-data/load-data! :materialize [& args] - (apply load-data/load-data-add-ids-chunked! args)) + (apply load-data/load-data-maybe-add-ids-chunked! args)) (defmethod tx/sorts-nil-first? :materialize [_driver _base-type]