From a9df5049c15ce91930929962a1edc296cd476658 Mon Sep 17 00:00:00 2001 From: Commelina Date: Fri, 11 Aug 2023 14:30:39 +0800 Subject: [PATCH] sql: turn on hstream_enable_schema flag by default (#1550) * sql: fix incorrect subquery planning * sql: turn on `hstream_enable_schema` flag by default --- Makefile | 2 +- common/hstream/hstream-common.cabal | 4 +- external/protocol | 2 +- hstream-processing/hstream-processing.cabal | 2 +- hstream-sql/etc/plan-test-cases.yaml | 24 +- hstream-sql/etc/syntax-test-cases.yaml | 332 ++++++++++++------ hstream-sql/hstream-sql.cabal | 2 +- hstream-sql/src/HStream/SQL/PlannerNew.hs | 5 +- .../src/HStream/SQL/PlannerNew/Types.hs | 4 +- hstream/hstream.cabal | 2 +- hstream/src/HStream/Client/SQLNew.hs | 4 +- 11 files changed, 247 insertions(+), 136 deletions(-) diff --git a/Makefile b/Makefile index adcf9bbba..fff00500c 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ HSTREAM_VERSION ?= $(or $(shell git describe --tag --abbrev=0 2>/dev/null), unkn HSTREAM_VERSION_COMMIT ?= $(or $(shell git rev-parse HEAD 2>/dev/null), unknown) ENGINE_VERSION ?= v1 -SQL_ENABLE_SCHEMA ?= false +SQL_ENABLE_SCHEMA ?= true ifeq ($(strip $(SQL_ENABLE_SCHEMA)), true) CABAL_BUILD_FLAGS += +hstream_enable_schema diff --git a/common/hstream/hstream-common.cabal b/common/hstream/hstream-common.cabal index 4f2242911..81c816f90 100644 --- a/common/hstream/hstream-common.cabal +++ b/common/hstream/hstream-common.cabal @@ -101,6 +101,7 @@ library , bytestring , casing >=0.1.4 , containers + , cryptonite , data-default , deepseq , digest @@ -115,6 +116,7 @@ library , hstream-common-base , http-client , http-types + , memory , optparse-applicative , primitive ^>=0.7 , proto3-suite @@ -132,8 +134,6 @@ library , zlib , zoovisitor , zstd - , memory - , cryptonite cxx-options: -std=c++17 cpp-options: -std=c++17 diff --git a/external/protocol b/external/protocol index c6efad1c6..1a61d13c2 160000 --- a/external/protocol +++ b/external/protocol @@ -1 +1 @@ -Subproject commit c6efad1c68b0b911e0ccca1b160b2e71320a7068 +Subproject commit 1a61d13c261900bf023afd851fcd1f4e3e446ca4 diff --git a/hstream-processing/hstream-processing.cabal b/hstream-processing/hstream-processing.cabal index 61c36621f..71aab981e 100644 --- a/hstream-processing/hstream-processing.cabal +++ b/hstream-processing/hstream-processing.cabal @@ -29,7 +29,7 @@ flag ReleaseBuild Enables all optimisations, leads to slower build time and better performance flag hstream_enable_schema - default: False + default: True description: Save schema with streams and enable schema-related pipelines diff --git a/hstream-sql/etc/plan-test-cases.yaml b/hstream-sql/etc/plan-test-cases.yaml index b4877ba9a..ec92ef110 100644 --- a/hstream-sql/etc/plan-test-cases.yaml +++ b/hstream-sql/etc/plan-test-cases.yaml @@ -4,29 +4,29 @@ testSuiteCases: - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: "Project (name=xNOTBETWEENSYMMETRIC0AND42, alias=xNOTBETWEENSYMMETRIC0AND42) - []\n Affiliate (name=xNOTBETWEENSYMMETRIC0AND42, expr=OpNotBetweenSymAnd(#(x), - ConstantInt 0, ConstantInt 42)) \n StreamScan s" + testCaseResult: |- + Project (OpNotBetweenSymAnd(0.#0,0,42)) | (#0_xNOTBETWEENSYMMETRIC0AND42:bool) + Scan s | (#0_x:int) testCaseStmts: - select x not between symmetric 0 and 42 from s; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: "Project (name=xNOTBETWEENSYMMETRIC0AND42, alias=xNOTBETWEENSYMMETRIC0AND42) - []\n Affiliate (name=xNOTBETWEENSYMMETRIC0AND42, expr=OpNotBetweenSymAnd(#(x), - ConstantInt 0, ConstantInt 42)) \n StreamScan s" + testCaseResult: |- + Project (OpNotBetweenSymAnd(0.#0,0,42)) | (#0_xNOTBETWEENSYMMETRIC0AND42:bool) + Scan s | (#0_x:int) testCaseStmts: - select x not between symmetric 0 and 42 from s; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: "Project (name=xNOTBETWEENSYMMETRIC0AND42, alias=xNOTBETWEENSYMMETRIC0AND42) - []\n Affiliate (name=xNOTBETWEENSYMMETRIC0AND42, expr=OpNotBetweenSymAnd(#(x), - ConstantInt 0, ConstantInt 42)) \n StreamScan s" + testCaseResult: |- + Project (OpNotBetweenSymAnd(0.#0,0,42)) | (#0_xNOTBETWEENSYMMETRIC0AND42:bool) + Scan s | (#0_x:int) testCaseStmts: - select x not between symmetric 0 and 42 from s; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: "Project (name=xNOTBETWEENSYMMETRIC0AND42, alias=xNOTBETWEENSYMMETRIC0AND42) - []\n Affiliate (name=xNOTBETWEENSYMMETRIC0AND42, expr=OpNotBetweenSymAnd(#(x), - ConstantInt 0, ConstantInt 42)) \n StreamScan s" + testCaseResult: |- + Project (OpNotBetweenSymAnd(0.#0,0,42)) | (#0_xNOTBETWEENSYMMETRIC0AND42:bool) + Scan s | (#0_x:int) testCaseStmts: - select x not between symmetric 0 and 42 from s; diff --git a/hstream-sql/etc/syntax-test-cases.yaml b/hstream-sql/etc/syntax-test-cases.yaml index 4762ee4f1..b94286b52 100644 --- a/hstream-sql/etc/syntax-test-cases.yaml +++ b/hstream-sql/etc/syntax-test-cases.yaml @@ -4,8 +4,14 @@ testSuiteCases: - testCaseFail: null testCaseLabel: ok example - testCaseResult: RQSelect (RSelect (RSel [RSelectProjectAll]) (RFrom (RTableRefSimple - "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["a","x","b"]] + { boundSel = BoundExprCol "a" "s" "a" 1, BoundExprCol "x" "s" "x" 0, BoundExprCol "b" "s" "b" 2 + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select * from s; - select*from s; @@ -26,31 +32,44 @@ testSuiteCases: - not SQL - testCaseFail: null testCaseLabel: deny unquoted dash in ident (1410) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprAccessJson - "c->>\"season_id\"" JOpLongArrow (RExprCol "c" Nothing "c") (RExprCol "\"season_id\"" - Nothing "season_id")) Nothing]) (RFrom (RTableRefSimple "production_changes" Nothing)) - RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["c->>'season_id'"]] + { boundSel = BoundExprAccessJson "c->>'season_id'" JOpLongArrow (BoundExprCol "c" "s" "c" 0) (BoundExprConst "'season_id'" "season_id") + , boundFrom = [SIMPLE] "production_changes" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - - select c->>"season_id" from production_changes EMIT CHANGES; - - select c ->> "season_id" from production_changes EMIT CHANGES; - - select c->> "season_id" from production_changes EMIT CHANGES; - - select c ->>"season_id" from production_changes EMIT CHANGES; + - select c->>'season_id' from production_changes EMIT CHANGES; + - select c ->> 'season_id' from production_changes EMIT CHANGES; + - select c->> 'season_id' from production_changes EMIT CHANGES; + - select c ->>'season_id' from production_changes EMIT CHANGES; - testCaseFail: null testCaseLabel: deny unquoted dash in ident (1410) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprAccessJson - "c->\"season_id\"" JOpArrow (RExprCol "c" Nothing "c") (RExprCol "\"season_id\"" - Nothing "season_id")) Nothing]) (RFrom (RTableRefSimple "production_changes" Nothing)) - RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["c->'season_id'"]] + { boundSel = BoundExprAccessJson "c->'season_id'" JOpArrow (BoundExprCol "c" "s" "c" 0) (BoundExprConst "'season_id'" "season_id") + , boundFrom = [SIMPLE] "production_changes" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - - select c ->"season_id" from production_changes EMIT CHANGES; - - select c -> "season_id" from production_changes EMIT CHANGES; - - select c->"season_id" from production_changes EMIT CHANGES; - - select c-> "season_id" from production_changes EMIT CHANGES; + - select c ->'season_id' from production_changes EMIT CHANGES; + - select c -> 'season_id' from production_changes EMIT CHANGES; + - select c->'season_id' from production_changes EMIT CHANGES; + - select c-> 'season_id' from production_changes EMIT CHANGES; - testCaseFail: null testCaseLabel: deny unquoted dash in ident (1410) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprBinOp "a-b" - OpSub (RExprCol "a" Nothing "a") (RExprCol "b" Nothing "b")) Nothing]) (RFrom - (RTableRefSimple "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["a-b"]] + { boundSel = BoundExprBinOp "a-b" OpSub (BoundExprCol "a" "s" "a" 1) (BoundExprCol "b" "s" "b" 2) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select a-b from s emit changes; - select a -b from s emit changes; @@ -58,24 +77,32 @@ testSuiteCases: - select a - b from s emit changes; - testCaseFail: null testCaseLabel: JSON operator should unquote string literal - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprAccessJson - "c->'season_id'" JOpArrow (RExprCol "c" Nothing "c") (RExprConst "'season_id'" - (ConstantText "season_id"))) Nothing]) (RFrom (RTableRefSimple "production_changes" - Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["c->'season_id'"]] + { boundSel = BoundExprAccessJson "c->'season_id'" JOpArrow (BoundExprCol "c" "s" "c" 0) (BoundExprConst "'season_id'" "season_id") + , boundFrom = [SIMPLE] "production_changes" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select c -> 'season_id' from production_changes EMIT CHANGES; - testCaseFail: null testCaseLabel: JSON operator should unquote string literal - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprAccessJson - "c->>'season_id'" JOpLongArrow (RExprCol "c" Nothing "c") (RExprConst "'season_id'" - (ConstantText "season_id"))) Nothing]) (RFrom (RTableRefSimple "production_changes" - Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["c->>'season_id'"]] + { boundSel = BoundExprAccessJson "c->>'season_id'" JOpLongArrow (BoundExprCol "c" "s" "c" 0) (BoundExprConst "'season_id'" "season_id") + , boundFrom = [SIMPLE] "production_changes" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select c ->> 'season_id' from production_changes EMIT CHANGES; - testCaseFail: null testCaseLabel: insert JSON - testCaseResult: 'RQInsert (RInsertRawOrJson "s" "{\"x\": {\"a\": 42, \"b\": null, - \"c\": \"str\", \"d\": true}}" RInsertRawOrJsonPayloadTypeJson)' + testCaseResult: 'BoundQInsert (BoundInsertRawOrJson "s" "{\"x\": {\"a\": 42, \"b\": + null, \"c\": \"str\", \"d\": true}}" BoundInsertPayloadTypeJson)' testCaseStmts: - 'insert into s values cast (''{"x": {"a": 42, "b": null, "c": "str", "d": true}}'' as jsonb);' @@ -83,169 +110,250 @@ testSuiteCases: jsonb;' - testCaseFail: null testCaseLabel: insert BYTEA - testCaseResult: RQInsert (RInsertRawOrJson "s" "binary_value" RInsertRawOrJsonPayloadTypeRaw) + testCaseResult: BoundQInsert (BoundInsertRawOrJson "s" "binary_value" BoundInsertPayloadTypeRaw) testCaseStmts: - insert into s values cast ('binary_value' as bytea); - 'insert into s values ''binary_value'' :: bytea;' - testCaseFail: null testCaseLabel: string function - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprBinOp "SPLIT('a.b.c',x)" - OpSplit (RExprConst "'a.b.c'" (ConstantText "a.b.c")) (RExprCol "x" Nothing "x")) - Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["SPLIT('a.b.c',x)"]] + { boundSel = BoundExprBinOp "SPLIT('a.b.c',x)" OpSplit (BoundExprConst "'a.b.c'" "a.b.c") (BoundExprCol "x" "s" "x" 0) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select SPLIT('a.b.c', x) from s emit changes; - testCaseFail: null testCaseLabel: BETWEEN - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xBETWEEN0AND42" - OpBetweenAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) (RExprConst - "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xBETWEEN0AND42"]] + { boundSel = BoundExprTerOp "xBETWEEN0AND42" OpBetweenAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x between 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: BETWEEN - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xNOTBETWEEN0AND42" - OpNotBetweenAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) (RExprConst - "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xNOTBETWEEN0AND42"]] + { boundSel = BoundExprTerOp "xNOTBETWEEN0AND42" OpNotBetweenAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x not between 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: BETWEEN - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xBETWEENSYMMETRIC0AND42" - OpBetweenSymAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) (RExprConst - "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xBETWEENSYMMETRIC0AND42"]] + { boundSel = BoundExprTerOp "xBETWEENSYMMETRIC0AND42" OpBetweenSymAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x between symmetric 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: BETWEEN - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xNOTBETWEENSYMMETRIC0AND42" - OpNotBetweenSymAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) - (RExprConst "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) - RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xNOTBETWEENSYMMETRIC0AND42"]] + { boundSel = BoundExprTerOp "xNOTBETWEENSYMMETRIC0AND42" OpNotBetweenSymAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x not between symmetric 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprUnaryOp "NOTxBETWEEN0AND42" - OpNot (RExprTerOp "xBETWEEN0AND42" OpBetweenAnd (RExprCol "x" Nothing "x") (RExprConst - "0" (ConstantInt 0)) (RExprConst "42" (ConstantInt 42)))) Nothing]) (RFrom (RTableRefSimple - "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["NOTxBETWEEN0AND42"]] + { boundSel = BoundExprUnaryOp "NOTxBETWEEN0AND42" OpNot (BoundExprTerOp "xBETWEEN0AND42" OpBetweenAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42)) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select not x between 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xNOTBETWEEN0AND42" - OpNotBetweenAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) (RExprConst - "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xNOTBETWEEN0AND42"]] + { boundSel = BoundExprTerOp "xNOTBETWEEN0AND42" OpNotBetweenAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x not between 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xBETWEENSYMMETRIC0AND42" - OpBetweenSymAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) (RExprConst - "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xBETWEENSYMMETRIC0AND42"]] + { boundSel = BoundExprTerOp "xBETWEENSYMMETRIC0AND42" OpBetweenSymAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x between symmetric 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: not(BETWEEN) - testCaseResult: RQPushSelect (RSelect (RSel [RSelectItemProject (RExprTerOp "xNOTBETWEENSYMMETRIC0AND42" - OpNotBetweenSymAnd (RExprCol "x" Nothing "x") (RExprConst "0" (ConstantInt 0)) - (RExprConst "42" (ConstantInt 42))) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) - RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQPushSelect BoundSelect [["xNOTBETWEENSYMMETRIC0AND42"]] + { boundSel = BoundExprTerOp "xNOTBETWEENSYMMETRIC0AND42" OpNotBetweenSymAnd (BoundExprCol "x" "s" "x" 0) (BoundExprConst "0" 0) (BoundExprConst "42" 42) + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select x not between symmetric 0 and 42 from s emit changes; - testCaseFail: null testCaseLabel: DATE syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprConst "DATE'2021-08-07'" - (ConstantDate 2021-08-07)) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["DATE'2021-08-07'"]] + { boundSel = BoundExprConst "DATE'2021-08-07'" 2021-08-07 + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select DATE '2021-08-07' from s; - testCaseFail: null testCaseLabel: TIME syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprConst "TIME'10:41:03'" - (ConstantTime 10:41:03)) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["TIME'10:41:03'"]] + { boundSel = BoundExprConst "TIME'10:41:03'" 10:41:03 + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select TIME '10:41:03' from s; - testCaseFail: null testCaseLabel: TIME syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprConst "TIME'01:02:03.456'" - (ConstantTime 01:02:03.456)) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["TIME'01:02:03.456'"]] + { boundSel = BoundExprConst "TIME'01:02:03.456'" 01:02:03.456 + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select TIME '01:02:03.456' from s; - testCaseFail: null testCaseLabel: TIMESTAMP syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprConst "TIMESTAMP'2023-06-30T12:30:45+02:00'" - (ConstantTimestamp 2023-06-30 12:30:45 +0200)) Nothing]) (RFrom (RTableRefSimple - "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["TIMESTAMP'2023-06-30T12:30:45+02:00'"]] + { boundSel = BoundExprConst "TIMESTAMP'2023-06-30T12:30:45+02:00'" 2023-06-30 12:30:45 +0200 + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select TIMESTAMP '2023-06-30T12:30:45+02:00' from s; - testCaseFail: null testCaseLabel: INTERVAL syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprConst "INTERVAL5SECOND" - (ConstantInterval P0MT5S)) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["INTERVAL5SECOND"]] + { boundSel = BoundExprConst "INTERVAL5SECOND" P0MT5S + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select INTERVAL 5 SECOND from s; - testCaseFail: null testCaseLabel: ARRAY syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprArray "{'aa','bb','cc'}" - [RExprConst "'aa'" (ConstantText "aa"),RExprConst "'bb'" (ConstantText "bb"),RExprConst - "'cc'" (ConstantText "cc")]) Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty - RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["{'aa','bb','cc'}"]] + { boundSel = BoundExprArray "{'aa','bb','cc'}" [BoundExprConst "'aa'" "aa",BoundExprConst "'bb'" "bb",BoundExprConst "'cc'" "cc"] + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select {'aa', 'bb', 'cc'} from s; - testCaseFail: null testCaseLabel: ARRAY syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprArray "{}" []) - Nothing]) (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["{}"]] + { boundSel = BoundExprArray "{}" [] + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select {} from s; - select { } from s; - testCaseFail: null testCaseLabel: ARRAY syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprArray "{{'aa','bb','cc'},{'aa','bb'},{'aa','bb','cc'}}" - [RExprArray "{'aa','bb','cc'}" [RExprConst "'aa'" (ConstantText "aa"),RExprConst - "'bb'" (ConstantText "bb"),RExprConst "'cc'" (ConstantText "cc")],RExprArray "{'aa','bb'}" - [RExprConst "'aa'" (ConstantText "aa"),RExprConst "'bb'" (ConstantText "bb")],RExprArray - "{'aa','bb','cc'}" [RExprConst "'aa'" (ConstantText "aa"),RExprConst "'bb'" (ConstantText - "bb"),RExprConst "'cc'" (ConstantText "cc")]]) Nothing]) (RFrom (RTableRefSimple - "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["{{'aa','bb','cc'},{'aa','bb'},{'aa','bb','cc'}}"]] + { boundSel = BoundExprArray "{{'aa','bb','cc'},{'aa','bb'},{'aa','bb','cc'}}" [BoundExprArray "{'aa','bb','cc'}" [BoundExprConst "'aa'" "aa",BoundExprConst "'bb'" "bb",BoundExprConst "'cc'" "cc"],BoundExprArray "{'aa','bb'}" [BoundExprConst "'aa'" "aa",BoundExprConst "'bb'" "bb"],BoundExprArray "{'aa','bb','cc'}" [BoundExprConst "'aa'" "aa",BoundExprConst "'bb'" "bb",BoundExprConst "'cc'" "cc"]] + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select {{'aa', 'bb', 'cc'}, {'aa', 'bb'}, {'aa', 'bb', 'cc'}} from s; - testCaseFail: null testCaseLabel: ARRAY syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprAccessArray "({'aa','bb','cc'})[0]" - (RExprArray "{'aa','bb','cc'}" [RExprConst "'aa'" (ConstantText "aa"),RExprConst - "'bb'" (ConstantText "bb"),RExprConst "'cc'" (ConstantText "cc")]) [0]) Nothing]) - (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["({'aa','bb','cc'})[0]"]] + { boundSel = BoundExprAccessArray "({'aa','bb','cc'})[0]" (BoundExprArray "{'aa','bb','cc'}" [BoundExprConst "'aa'" "aa",BoundExprConst "'bb'" "bb",BoundExprConst "'cc'" "cc"]) [0] + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - select ({'aa', 'bb', 'cc'})[0] from s; - testCaseFail: null testCaseLabel: ARRAY syntax - testCaseResult: RQSelect (RSelect (RSel [RSelectItemProject (RExprAccessArray "({'aa','bb','cc'})[0:2]" - (RExprArray "{'aa','bb','cc'}" [RExprConst "'aa'" (ConstantText "aa"),RExprConst - "'bb'" (ConstantText "bb"),RExprConst "'cc'" (ConstantText "cc")]) [0:2]) Nothing]) - (RFrom (RTableRefSimple "s" Nothing)) RWhereEmpty RGroupByEmpty RHavingEmpty) + testCaseResult: |- + BoundQSelect BoundSelect [["({'aa','bb','cc'})[0:2]"]] + { boundSel = BoundExprAccessArray "({'aa','bb','cc'})[0:2]" (BoundExprArray "{'aa','bb','cc'}" [BoundExprConst "'aa'" "aa",BoundExprConst "'bb'" "bb",BoundExprConst "'cc'" "cc"]) [0:2] + , boundFrom = [SIMPLE] "s" + , boundWhere = BoundWhereEmpty + , boundGroupBy = BoundGroupByEmpty + , boundHaving = BoundHavingEmpty + } testCaseStmts: - 'select ({''aa'', ''bb'', ''cc''})[0 : 2] from s;' - testCaseFail: null testCaseLabel: Create Connectors - testCaseResult: RQCreate (RCreateConnector "SOURCE" "source01" "mysql" False (RConnectorOptions - (fromList [("port",Object (fromList [("contents",Number 3306.0),("tag",String - "ConstantInt")])),("host",Object (fromList [("contents",String "mysql-s1"),("tag",String - "ConstantText")])),("password",Object (fromList [("contents",String "password"),("tag",String - "ConstantText")])),("user",Object (fromList [("contents",String "root"),("tag",String - "ConstantText")])),("database",Object (fromList [("contents",String "d1"),("tag",String - "ConstantText")])),("table",Object (fromList [("contents",String "person"),("tag",String - "ConstantText")])),("stream",Object (fromList [("contents",String "stream01"),("tag",String - "ConstantText")]))]))) + testCaseResult: BoundQCreate (BoundCreateConnector "SOURCE" "source01" "mysql" False + (BoundConnectorOptions (fromList [("port",Object (fromList [("contents",Number + 3306.0),("tag",String "ConstantInt")])),("host",Object (fromList [("contents",String + "mysql-s1"),("tag",String "ConstantText")])),("password",Object (fromList [("contents",String + "password"),("tag",String "ConstantText")])),("user",Object (fromList [("contents",String + "root"),("tag",String "ConstantText")])),("database",Object (fromList [("contents",String + "d1"),("tag",String "ConstantText")])),("table",Object (fromList [("contents",String + "person"),("tag",String "ConstantText")])),("stream",Object (fromList [("contents",String + "stream01"),("tag",String "ConstantText")]))]))) testCaseStmts: - create source connector source01 from mysql with (host = 'mysql-s1', port = 3306, user = 'root', password = 'password', database = 'd1', table = 'person', "stream" diff --git a/hstream-sql/hstream-sql.cabal b/hstream-sql/hstream-sql.cabal index f02605f39..8ae6bc1a8 100644 --- a/hstream-sql/hstream-sql.cabal +++ b/hstream-sql/hstream-sql.cabal @@ -36,7 +36,7 @@ flag hstream_use_v2_engine description: Use v2 processing engine flag hstream_enable_schema - default: False + default: True description: Save schema with streams and enable schema-related pipelines diff --git a/hstream-sql/src/HStream/SQL/PlannerNew.hs b/hstream-sql/src/HStream/SQL/PlannerNew.hs index ec224f118..5961ca0b1 100644 --- a/hstream-sql/src/HStream/SQL/PlannerNew.hs +++ b/hstream-sql/src/HStream/SQL/PlannerNew.hs @@ -62,7 +62,7 @@ instance Plan BoundTableRef where setSchemaStreamId 0 (relationExprSchema relationExpr) let ctx = PlanContext (IntMap.singleton 0 schema) mempty put ctx - return relationExpr + return $ setRelationExprSchema schema relationExpr -- Note: As `HStream.SQL.Binder.Select` says, 'BoundTableRefWindowed' is -- only used when binding 'SELECT'. So we can ignore the window part here @@ -91,8 +91,7 @@ instance Plan BoundTableRef where (scalarExpr,_) <- plan expr -- 1-out! - let schema = setSchemaStream name $ - setSchemaStreamId 0 $ Schema + let schema = setSchemaStreamId 0 $ Schema { schemaOwner = name , schemaColumns = schemaColumns schema1 <:+:> schemaColumns schema2 diff --git a/hstream-sql/src/HStream/SQL/PlannerNew/Types.hs b/hstream-sql/src/HStream/SQL/PlannerNew/Types.hs index 7c80df7f6..20429d327 100644 --- a/hstream-sql/src/HStream/SQL/PlannerNew/Types.hs +++ b/hstream-sql/src/HStream/SQL/PlannerNew/Types.hs @@ -141,7 +141,9 @@ lookupColumn (PlanContext m baseRefs) streamName colId = in (fmap (\(n,catalog) -> (i,n,catalog)) catalogTup_m) <|> acc ) Nothing (IntMap.toList m) where go (s,i) = case HM.lookup (s,i) baseRefs of - Just (s',i') -> go (s',i') + Just (s',i') -> if (s,i) == (s',i') + then (s,i) + else go (s',i') Nothing -> (s,i) -- | Lookup a certain column name in the planning context. Return the index of diff --git a/hstream/hstream.cabal b/hstream/hstream.cabal index 987bc38d4..d6b2af70a 100644 --- a/hstream/hstream.cabal +++ b/hstream/hstream.cabal @@ -37,7 +37,7 @@ flag hstream_use_v2_engine description: Use v2 processing engine flag hstream_enable_schema - default: False + default: True description: Save schema with streams and enable schema-related pipelines diff --git a/hstream/src/HStream/Client/SQLNew.hs b/hstream/src/HStream/Client/SQLNew.hs index f9cd17dcc..ca454398b 100644 --- a/hstream/src/HStream/Client/SQLNew.hs +++ b/hstream/src/HStream/Client/SQLNew.hs @@ -54,6 +54,7 @@ import HStream.Client.Types (HStreamCliContext (..), Resource (..)) import HStream.Client.Utils (calculateShardId, dropPlanToResType) +import HStream.Common.Types (hashShardKey) import HStream.Server.HStreamApi (CommandQuery (..), CommandQueryResponse (..), HStreamApi (..), @@ -152,7 +153,8 @@ commandExec HStreamSqlContext{hstreamCliContext = cliCtx@HStreamCliContext{..},. result <- execute cliCtx $ listShards sName case result of Just (API.ListShardsResponse shards) -> do - case calculateShardId "" (V.toList shards) of + let shardKey = hashShardKey "" + case calculateShardId shardKey (V.toList shards) of Nothing -> putStrLn "Failed to calculate shard id" Just sid -> executeWithLookupResource_ cliCtx (Resource ResShard (T.pack $ show sid)) (retry retryLimit retryInterval $ insertIntoStream sName sid (insertType == JsonFormat) payload) Nothing -> putStrLn "No shards found"