diff --git a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider index 0d8a2c58e5c..ddb5edfd816 100644 --- a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider +++ b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider @@ -16,6 +16,7 @@ # org.apache.kyuubi.engine.jdbc.doris.DorisConnectionProvider +org.apache.kyuubi.engine.jdbc.impala.ImpalaConnectionProvider org.apache.kyuubi.engine.jdbc.mysql.MySQLConnectionProvider org.apache.kyuubi.engine.jdbc.phoenix.PhoenixConnectionProvider org.apache.kyuubi.engine.jdbc.postgresql.PostgreSQLConnectionProvider diff --git a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect index c5a75ec9c9f..df3adc309a3 100644 --- a/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect +++ b/externals/kyuubi-jdbc-engine/src/main/resources/META-INF/services/org.apache.kyuubi.engine.jdbc.dialect.JdbcDialect @@ -16,6 +16,7 @@ # org.apache.kyuubi.engine.jdbc.dialect.DorisDialect +org.apache.kyuubi.engine.jdbc.dialect.ImpalaDialect org.apache.kyuubi.engine.jdbc.dialect.MySQLDialect org.apache.kyuubi.engine.jdbc.dialect.PhoenixDialect org.apache.kyuubi.engine.jdbc.dialect.PostgreSQLDialect diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/JdbcConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/JdbcConnectionProvider.scala index d6ff316ab6e..e247aa3949d 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/JdbcConnectionProvider.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/JdbcConnectionProvider.scala @@ -30,7 +30,9 @@ abstract class JdbcConnectionProvider extends SupportServiceLoader with Logging val driverClass: String - def canHandle(providerClass: String): Boolean + def canHandle(providerClass: String): Boolean = { + driverClass.equalsIgnoreCase(providerClass) + } def getConnection(kyuubiConf: KyuubiConf): Connection = { val properties = new Properties() diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/ImpalaDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/ImpalaDialect.scala new file mode 100644 index 00000000000..2c08655ad06 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/ImpalaDialect.scala @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.dialect + +import java.util + +import org.apache.commons.lang3.StringUtils + +import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.engine.jdbc.impala.{ImpalaSchemaHelper, ImpalaTRowSetGenerator} +import org.apache.kyuubi.engine.jdbc.schema.{JdbcTRowSetGenerator, SchemaHelper} +import org.apache.kyuubi.session.Session + +class ImpalaDialect extends JdbcDialect { + + override def getTablesQuery( + catalog: String, + schema: String, + tableName: String, + tableTypes: util.List[String]): String = { + if (isPattern(schema)) { + throw KyuubiSQLException.featureNotSupported("Pattern-like schema names not supported") + } + + val query = new StringBuilder("show tables ") + + if (StringUtils.isNotEmpty(schema) && !isWildcardSetByKyuubi(schema)) { + query.append(s"in $schema ") + } + + if (StringUtils.isNotEmpty(tableName)) { + query.append(s"like '${toImpalaRegex(tableName)}'") + } + + query.toString() + } + + override def getColumnsQuery( + session: Session, + catalogName: String, + schemaName: String, + tableName: String, + columnName: String): String = { + if (StringUtils.isEmpty(tableName)) { + throw KyuubiSQLException("Table name should not be empty") + } + + if (isPattern(schemaName)) { + throw KyuubiSQLException.featureNotSupported("Pattern-like schema names not supported") + } + + if (isPattern(tableName)) { + throw KyuubiSQLException.featureNotSupported("Pattern-like table names not supported") + } + + val query = new StringBuilder("show column stats ") + + if (StringUtils.isNotEmpty(schemaName) && !isWildcardSetByKyuubi(schemaName)) { + query.append(s"$schemaName.") + } + + query.append(tableName) + query.toString() + } + + override def getTRowSetGenerator(): JdbcTRowSetGenerator = new ImpalaTRowSetGenerator + + override def getSchemaHelper(): SchemaHelper = new ImpalaSchemaHelper + + override def name(): String = "impala" + + private def isPattern(value: String): Boolean = { + value != null && !isWildcardSetByKyuubi(value) && value.contains("*") + } + + private def isWildcardSetByKyuubi(pattern: String): Boolean = pattern == "%" + + private def toImpalaRegex(pattern: String): String = { + pattern.replace("%", "*") + } +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaConnectionProvider.scala new file mode 100644 index 00000000000..232d723fba4 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaConnectionProvider.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.connection.JdbcConnectionProvider + +class ImpalaConnectionProvider extends JdbcConnectionProvider { + + override val name: String = classOf[ImpalaConnectionProvider].getName + + override val driverClass: String = ImpalaConnectionProvider.driverClass +} + +object ImpalaConnectionProvider { + // we should use kyuubi hive driver instead of original hive one in order + // to get fixed getMoreResults() + val driverClass: String = "org.apache.kyuubi.jdbc.KyuubiHiveDriver" +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaSchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaSchemaHelper.scala new file mode 100644 index 00000000000..f0d1cd68c84 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaSchemaHelper.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.schema.SchemaHelper +import org.apache.kyuubi.shaded.hive.service.rpc.thrift.TTypeId + +class ImpalaSchemaHelper extends SchemaHelper { + override protected def floatToTTypeId: TTypeId = { + TTypeId.DOUBLE_TYPE + } + + override protected def realToTTypeId: TTypeId = { + TTypeId.DOUBLE_TYPE + } +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaTRowSetGenerator.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaTRowSetGenerator.scala new file mode 100644 index 00000000000..85ad0cdc8df --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaTRowSetGenerator.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.schema.DefaultJdbcTRowSetGenerator +import org.apache.kyuubi.shaded.hive.service.rpc.thrift.{TColumn, TColumnValue} + +class ImpalaTRowSetGenerator extends DefaultJdbcTRowSetGenerator { + override def toFloatTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = + asDoubleTColumn(rows, ordinal) + + override def toFloatTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = + asDoubleTColumnValue(row, ordinal) + + override def toRealTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = + asDoubleTColumn(rows, ordinal) + + override def toRealTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = + asDoubleTColumnValue(row, ordinal) +} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/mysql/MySQL8ConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/mysql/MySQL8ConnectionProvider.scala index f96dff10784..6996df861b4 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/mysql/MySQL8ConnectionProvider.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/mysql/MySQL8ConnectionProvider.scala @@ -23,11 +23,6 @@ class MySQL8ConnectionProvider extends JdbcConnectionProvider { override val name: String = classOf[MySQL8ConnectionProvider].getName override val driverClass: String = MySQL8ConnectionProvider.driverClass - - override def canHandle(providerClass: String): Boolean = { - driverClass.equalsIgnoreCase(providerClass) - } - } object MySQL8ConnectionProvider { diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala index 4292c320b30..af9e9a10274 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/ExecuteStatement.scala @@ -76,8 +76,9 @@ class ExecuteStatement( }) } else { warn(s"Execute in full collect mode") - jdbcStatement.closeOnCompletion() - new ArrayFetchIterator(resultSetWrapper.toArray()) + val arrayIter = new ArrayFetchIterator(resultSetWrapper.toArray()) + jdbcStatement.close() + arrayIter } } else { schema = Schema(List[Column]( diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixConnectionProvider.scala index 95c6f1c0040..591ca472007 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixConnectionProvider.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixConnectionProvider.scala @@ -23,9 +23,4 @@ class PhoenixConnectionProvider extends JdbcConnectionProvider { override val name: String = classOf[PhoenixConnectionProvider].getName override val driverClass: String = "org.apache.phoenix.queryserver.client.Driver" - - override def canHandle(providerClass: String): Boolean = { - driverClass.equalsIgnoreCase(providerClass) - } - } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/postgresql/PostgreSQLConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/postgresql/PostgreSQLConnectionProvider.scala index a11e5776828..dcd8a878659 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/postgresql/PostgreSQLConnectionProvider.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/postgresql/PostgreSQLConnectionProvider.scala @@ -24,8 +24,4 @@ class PostgreSQLConnectionProvider extends JdbcConnectionProvider { override val driverClass: String = "org.postgresql.Driver" - override def canHandle(providerClass: String): Boolean = { - driverClass.equalsIgnoreCase(providerClass) - } - } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala index 2c9ddd6da3e..ec5de39496b 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/DefaultJdbcTRowSetGenerator.scala @@ -33,9 +33,11 @@ class DefaultJdbcTRowSetGenerator extends JdbcTRowSetGenerator { case INTEGER => toIntegerTColumn(rows, ordinal) case BIGINT => toBigIntTColumn(rows, ordinal) case REAL => toRealTColumn(rows, ordinal) + case FLOAT => toFloatTColumn(rows, ordinal) case DOUBLE => toDoubleTColumn(rows, ordinal) case CHAR => toCharTColumn(rows, ordinal) case VARCHAR => toVarcharTColumn(rows, ordinal) + case BOOLEAN => toBooleanTColumn(rows, ordinal) case _ => toDefaultTColumn(rows, ordinal, sqlType) } @@ -47,9 +49,11 @@ class DefaultJdbcTRowSetGenerator extends JdbcTRowSetGenerator { case INTEGER => toIntegerTColumnValue(row, ordinal) case BIGINT => toBigIntTColumnValue(row, ordinal) case REAL => toRealTColumnValue(row, ordinal) + case FLOAT => toFloatTColumnValue(row, ordinal) case DOUBLE => toDoubleTColumnValue(row, ordinal) case CHAR => toCharTColumnValue(row, ordinal) case VARCHAR => toVarcharTColumnValue(row, ordinal) + case BOOLEAN => toBooleanTColumnValue(row, ordinal) case otherType => toDefaultTColumnValue(row, ordinal, otherType) } } @@ -63,6 +67,9 @@ class DefaultJdbcTRowSetGenerator extends JdbcTRowSetGenerator { def toBitTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = asBooleanTColumn(rows, ordinal) + def toBooleanTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = + asBooleanTColumn(rows, ordinal) + def toTinyIntTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = asShortTColumn(rows, ordinal) @@ -78,6 +85,9 @@ class DefaultJdbcTRowSetGenerator extends JdbcTRowSetGenerator { def toRealTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = asFloatTColumn(rows, ordinal) + def toFloatTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = + asFloatTColumn(rows, ordinal) + def toDoubleTColumn(rows: Seq[Seq[_]], ordinal: Int): TColumn = asDoubleTColumn(rows, ordinal) @@ -92,6 +102,9 @@ class DefaultJdbcTRowSetGenerator extends JdbcTRowSetGenerator { def toBitTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = asBooleanTColumnValue(row, ordinal) + def toBooleanTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = + asBooleanTColumnValue(row, ordinal) + def toTinyIntTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = asShortTColumnValue(row, ordinal) @@ -107,6 +120,9 @@ class DefaultJdbcTRowSetGenerator extends JdbcTRowSetGenerator { def toRealTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = asFloatTColumnValue(row, ordinal) + def toFloatTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = + asFloatTColumnValue(row, ordinal) + def toDoubleTColumnValue(row: Seq[_], ordinal: Int): TColumnValue = asDoubleTColumnValue(row, ordinal) diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala index 6b39bb3dbe4..16d46fc36f6 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala @@ -100,6 +100,12 @@ abstract class SchemaHelper { case Types.DECIMAL => decimalToTTypeId + case Types.FLOAT => + floatToTTypeId + + case Types.BOOLEAN => + booleanToTTypeId + // TODO add more type support case _ => defaultToTTypeId @@ -109,6 +115,10 @@ abstract class SchemaHelper { TTypeId.BOOLEAN_TYPE } + protected def booleanToTTypeId: TTypeId = { + TTypeId.BOOLEAN_TYPE + } + protected def tinyIntToTTypeId: TTypeId = { TTypeId.TINYINT_TYPE } @@ -129,6 +139,10 @@ abstract class SchemaHelper { TTypeId.FLOAT_TYPE } + protected def floatToTTypeId: TTypeId = { + TTypeId.FLOAT_TYPE + } + protected def doubleToTTypeId: TTypeId = { TTypeId.DOUBLE_TYPE } diff --git a/externals/kyuubi-jdbc-engine/src/test/resources/impala-compose.yml b/externals/kyuubi-jdbc-engine/src/test/resources/impala-compose.yml new file mode 100644 index 00000000000..4be34042b0f --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/resources/impala-compose.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# modified compose file from Impala repo, see +# https://github.com/apache/impala/blob/4.3.0/docker/quickstart.yml +version: "3.5" +services: + metastore: + image: apache/impala:4.0.0-impala_quickstart_hms + command: ["hms"] + ports: + - "9083" + volumes: + # Volume used to store Apache Derby database. + - impala-data:/var/lib/hive + # Warehouse directory. HMS does file operations so needs access to the + # shared volume. + - impala-data:/user/hive/warehouse + - ./impala_conf:/opt/hive/conf:ro + + statestored: + image: apache/impala:4.0.0-statestored + ports: + - "25010" + command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1"] + volumes: + - ./impala_conf:/opt/impala/conf:ro + + catalogd: + depends_on: + - statestored + - metastore + image: apache/impala:4.0.0-catalogd + ports: + - "25020" + command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1", + "-hms_event_polling_interval_s=1", "-invalidate_tables_timeout_s=999999"] + volumes: + # Warehouse directory. Catalog does file operations so needs access to the + # shared volume. + - impala-data:/user/hive/warehouse + - ./impala_conf:/opt/impala/conf:ro + + impalad: + image: apache/impala:4.0.0-impalad_coord_exec + depends_on: + - statestored + - catalogd + ports: + - "21050:21050" + command: [ "-v=1", + "-redirect_stdout_stderr=false", "-logtostderr", + "-mt_dop_auto_fallback=true", + "-default_query_options=mt_dop=4,default_file_format=parquet,default_transactional_type=insert_only", + "-mem_limit=1500mb"] + environment: + # Keep the Java heap small to preserve memory for query execution. + - JAVA_TOOL_OPTIONS="-Xmx1g" + volumes: + - impala-data:/user/hive/warehouse + - ./impala_conf:/opt/impala/conf:ro + +volumes: + impala-data: + +networks: + default: + name: default-kyuubi-impala-test diff --git a/externals/kyuubi-jdbc-engine/src/test/resources/impala_conf/hive-site.xml b/externals/kyuubi-jdbc-engine/src/test/resources/impala_conf/hive-site.xml new file mode 100644 index 00000000000..55fa54ef1f8 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/resources/impala_conf/hive-site.xml @@ -0,0 +1,72 @@ + + + + + + + + hive.metastore.dml.events + true + + + + hive.metastore.event.db.notification.api.auth + false + + + hive.metastore.uris + thrift://metastore:9083 + + + + hive.metastore.warehouse.dir + /user/hive/warehouse/managed + + + hive.metastore.warehouse.external.dir + /user/hive/warehouse/external + + + + hive.support.concurrency + true + + + + hive.txn.manager + org.apache.hadoop.hive.ql.lockmgr.DbTxnManager + + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + + javax.jdo.option.ConnectionURL + jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true + + + + hive.stats.autogather + false + + diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/DialectSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/DialectSuite.scala new file mode 100644 index 00000000000..7f90a609437 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/DialectSuite.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException} +import org.apache.kyuubi.engine.jdbc.dialect.ImpalaDialect +import org.apache.kyuubi.engine.jdbc.impala.DialectSuite.{SCHEME_NAME, TABLE_NAME} + +class DialectSuite extends KyuubiFunSuite { + + private val dialect: ImpalaDialect = new ImpalaDialect() + + test("impala dialect - get tables query") { + val expectedQuery = "show tables" + val actualQuery = dialect.getTablesQuery(null, null, null, null) + + assert(expectedQuery == actualQuery.trim) + } + + test("impala dialect - get tables query by scheme") { + val expectedQuery = s"show tables in $SCHEME_NAME" + val actualQuery = dialect.getTablesQuery(null, SCHEME_NAME, null, null) + + assert(expectedQuery == actualQuery.trim) + } + + test("impala dialect - get tables query by table name") { + val expectedQuery = s"show tables like '$TABLE_NAME'" + val queryWithTableName = dialect.getTablesQuery(null, null, TABLE_NAME, null) + + assert(expectedQuery == queryWithTableName.trim) + + // kyuubi injects '%' in case if schema is null + val queryWithWildcardScheme = dialect.getTablesQuery(null, "%", TABLE_NAME, null) + + assert(expectedQuery == queryWithWildcardScheme.trim) + } + + test("impala dialect - get tables query by scheme and table name") { + val expectedQuery = s"show tables in $SCHEME_NAME like 'test*'" + val actualQuery = dialect.getTablesQuery(null, SCHEME_NAME, "test*", null) + + assert(expectedQuery == actualQuery.trim) + } + + test("impala dialect - fail get tables if pattern-like scheme provided") { + val exception = intercept[KyuubiSQLException] { + dialect.getTablesQuery(null, "*scheme*", TABLE_NAME, null) + } + assert(exception.getMessage == "Pattern-like schema names not supported") + } + + test("impala dialect - get columns query by table name") { + val expectedQuery = s"show column stats $TABLE_NAME" + val actualQuery = dialect.getColumnsQuery(null, null, null, TABLE_NAME, null) + + assert(expectedQuery == actualQuery.trim) + } + + test("impala dialect - get columns query by scheme and table name") { + val expectedQuery = s"show column stats $SCHEME_NAME.$TABLE_NAME" + val actualQuery = dialect.getColumnsQuery(null, null, SCHEME_NAME, TABLE_NAME, null) + + assert(expectedQuery == actualQuery.trim) + } + + test("impala dialect - fail get columns if pattern-like scheme provided") { + val exception = intercept[KyuubiSQLException] { + dialect.getColumnsQuery(null, null, "*scheme*", TABLE_NAME, null) + } + assert(exception.getMessage == "Pattern-like schema names not supported") + } + + test("impala dialect - fail get columns if pattern-like table provided") { + val exception = intercept[KyuubiSQLException] { + dialect.getColumnsQuery(null, null, null, "*test*", null) + } + assert(exception.getMessage == "Pattern-like table names not supported") + } + + test("impala dialect - fail get columns if table name not provided") { + val exception = intercept[KyuubiSQLException] { + dialect.getColumnsQuery(null, null, null, null, null) + } + assert(exception.getMessage == "Table name should not be empty") + } +} + +object DialectSuite { + val TABLE_NAME = "test_table" + val SCHEME_NAME = "test_db" +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaOperationSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaOperationSuite.scala new file mode 100644 index 00000000000..ae3336d5a0a --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/ImpalaOperationSuite.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import java.sql.ResultSet + +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.operation.HiveJDBCTestHelper + +abstract class ImpalaOperationSuite extends WithImpalaEngine with HiveJDBCTestHelper { + test("impala - get tables") { + withJdbcStatement() { statement => + val meta = statement.getConnection.getMetaData + + statement.execute("create table test1(id bigint)") + statement.execute("create table test2(id bigint)") + statement.execute("create database db1") + statement.execute("create table db1.test3(id bigint)") + + var tables = meta.getTables(null, null, "test1", null) + while (tables.next()) { + assert(tables.getString(1) == "test1") + } + + tables = meta.getTables(null, null, "test2", null) + while (tables.next()) { + assert(tables.getString(1) == "test2") + } + + tables = meta.getTables(null, null, "test*", null) + + val actualTables = ArrayBuffer[String]() + while (tables.next()) { + actualTables += tables.getString(1) + } + assert(ArrayBuffer("test1", "test2") == actualTables) + + tables = meta.getTables(null, "db1", "test*", null) + while (tables.next()) { + assert(tables.getString(1) == "test3") + } + + statement.execute("drop table test1") + statement.execute("drop table test2") + statement.execute("drop table db1.test3") + statement.execute("drop database db1") + } + } + + test("impala - get columns") { + case class Column(name: String, columnType: String) + + def buildColumn(resultSet: ResultSet): Column = { + val columnName = resultSet.getString("Column") + val columnType = resultSet.getString("Type") + Column(columnName, columnType) + } + + withJdbcStatement() { statement => + val metadata = statement.getConnection.getMetaData + statement.execute("create table if not exists test1" + + "(id bigint, str1 string, str2 string, age int)") + + statement.execute("create database db1") + statement.execute("create table if not exists db1.test2" + + "(id bigint, str1 string)") + + val resultBuffer = ArrayBuffer[Column]() + val resultSet1 = metadata.getColumns(null, null, "test1", null) + while (resultSet1.next()) { + resultBuffer += buildColumn(resultSet1) + } + + assert(resultBuffer.contains(Column("id", "BIGINT"))) + assert(resultBuffer.contains(Column("str1", "STRING"))) + assert(resultBuffer.contains(Column("str2", "STRING"))) + assert(resultBuffer.contains(Column("age", "INT"))) + + resultBuffer.clear() + + val resultSet2 = metadata.getColumns(null, "db1", "test2", null) + while (resultSet2.next()) { + resultBuffer += buildColumn(resultSet2) + } + assert(resultBuffer.contains(Column("id", "BIGINT"))) + assert(resultBuffer.contains(Column("str1", "STRING"))) + + statement.execute("drop table test1") + statement.execute("drop table db1.test2") + statement.execute("drop database db1") + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/OperationWithImpalaEngineSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/OperationWithImpalaEngineSuite.scala new file mode 100644 index 00000000000..87fa49745e5 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/OperationWithImpalaEngineSuite.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.jdbc.connection.ConnectionProvider +import org.apache.kyuubi.operation.HiveJDBCTestHelper +import org.apache.kyuubi.shaded.hive.service.rpc.thrift.{TGetInfoReq, TGetInfoType} + +class OperationWithImpalaEngineSuite extends ImpalaOperationSuite with HiveJDBCTestHelper { + + override protected def jdbcUrl: String = jdbcConnectionUrl + + test("impala - test for Jdbc engine getInfo") { + val metaData = ConnectionProvider.create(kyuubiConf).getMetaData + + withSessionConf(Map(KyuubiConf.SERVER_INFO_PROVIDER.key -> "ENGINE"))()() { + withSessionHandle { (client, handle) => + val req = new TGetInfoReq() + req.setSessionHandle(handle) + req.setInfoType(TGetInfoType.CLI_DBMS_NAME) + assert(client.GetInfo(req).getInfoValue.getStringValue == metaData.getDatabaseProductName) + + val req2 = new TGetInfoReq() + req2.setSessionHandle(handle) + req2.setInfoType(TGetInfoType.CLI_DBMS_VER) + assert( + client.GetInfo(req2).getInfoValue.getStringValue == metaData.getDatabaseProductVersion) + + val req3 = new TGetInfoReq() + req3.setSessionHandle(handle) + req3.setInfoType(TGetInfoType.CLI_MAX_COLUMN_NAME_LEN) + assert(client.GetInfo(req3).getInfoValue.getLenValue == metaData.getMaxColumnNameLength) + } + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/SessionSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/SessionSuite.scala new file mode 100644 index 00000000000..7154e9cea20 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/SessionSuite.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.operation.HiveJDBCTestHelper + +class SessionSuite extends WithImpalaEngine with HiveJDBCTestHelper { + + test("impala - test session") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery( + "select '1' as id") + val metadata = resultSet.getMetaData + for (i <- 1 to metadata.getColumnCount) { + assert(metadata.getColumnName(i) == "id") + } + while (resultSet.next()) { + val id = resultSet.getObject(1) + assert(id == "1") + } + } + } + + override protected def jdbcUrl: String = jdbcConnectionUrl +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/StatementSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/StatementSuite.scala new file mode 100644 index 00000000000..ed6d7e4f0bd --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/StatementSuite.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import java.sql.{Date, Timestamp} + +import org.apache.kyuubi.operation.HiveJDBCTestHelper + +class StatementSuite extends WithImpalaEngine with HiveJDBCTestHelper { + + test("impala - test select") { + withJdbcStatement("test1") { statement => + statement.execute("create table test1(id bigint, " + + "name string, age integer)") + statement.execute("insert into test1 values(1, 'a', 11)") + + val resultSet1 = statement.executeQuery("select * from test1") + while (resultSet1.next()) { + val id = resultSet1.getObject(1) + assert(id == 1) + val name = resultSet1.getObject(2) + assert(name == "a") + val age = resultSet1.getObject(3) + assert(age == 11) + } + } + } + + test("impala - test types") { + withJdbcStatement("type_test") { statement => + statement.execute("create table type_test(" + + "id bigint, " + + "smallint_col smallint, " + + "int_col int, " + + "bigint_col bigint, " + + "date_col date, " + + "timestamp_col timestamp, " + + "char_col char(10), " + + "varchar_col varchar(255), " + + "boolean_col boolean, " + + "double_col double, " + + "real_col real, " + + "string_col STRING " + + ")") + + statement.execute("insert into type_test" + + "(id, " + + "smallint_col, " + + "int_col, " + + "bigint_col, " + + "date_col, " + + "timestamp_col, " + + "char_col, " + + "varchar_col, " + + "boolean_col, " + + "double_col, " + + "real_col," + + "string_col) " + + "VALUES (1, " + + "2, " + + "3, " + + "4, " + + "'2022-05-08', " + + "'2022-05-08 17:47:45'," + + "CAST('a' AS char(10)), " + + "CAST('Hello' AS varchar(255)), " + + "true, " + + "8.8, " + + "9.9, " + + "'test_str' " + + ")") + + val resultSet1 = statement.executeQuery("select * from type_test") + while (resultSet1.next()) { + assert(resultSet1.getObject(1) == 1) + assert(resultSet1.getObject(2) == 2) + assert(resultSet1.getObject(3) == 3) + assert(resultSet1.getObject(4) == 4) + assert(resultSet1.getObject(5) == Date.valueOf("2022-05-08")) + assert(resultSet1.getObject(6) == Timestamp.valueOf("2022-05-08 17:47:45")) + assert(resultSet1.getString(7).trim == "a") + assert(resultSet1.getObject(8) == "Hello") + assert(resultSet1.getObject(9) == true) + assert(resultSet1.getObject(10) == 8.8) + assert(resultSet1.getObject(11) == 9.9) + assert(resultSet1.getObject(12) == "test_str") + } + } + } + + override protected def jdbcUrl: String = jdbcConnectionUrl +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaContainer.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaContainer.scala new file mode 100644 index 00000000000..4aaa8871141 --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaContainer.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import java.io.File + +import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService} +import org.testcontainers.containers.wait.strategy.Wait + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer + +trait WithImpalaContainer extends WithJdbcServerContainer { + private val METASTORE_SERVICE_NAME = "metastore" + private val METASTORE_PORT = 9083 + + private val STATESTORE_SERVICE_NAME = "statestored" + private val STATESTORE_PORT = 25010 + + private val CATALOGD_SERVICE_NAME = "catalogd" + private val CATALOGD_PORT = 25020 + + private val IMPALAD_SERVICE_NAME = "impalad" + private val IMPALAD_PORT = 21050 + + override val containerDef: DockerComposeContainer.Def = + DockerComposeContainer + .Def( + composeFiles = new File(Utils.getContextOrKyuubiClassLoader + .getResource("impala-compose.yml").toURI), + exposedServices = Seq[ExposedService]( + ExposedService( + METASTORE_SERVICE_NAME, + METASTORE_PORT, + waitStrategy = Wait.forListeningPort), + ExposedService( + STATESTORE_SERVICE_NAME, + STATESTORE_PORT, + waitStrategy = Wait.forListeningPort), + ExposedService( + CATALOGD_SERVICE_NAME, + CATALOGD_PORT, + waitStrategy = Wait.forListeningPort), + ExposedService( + IMPALAD_SERVICE_NAME, + IMPALAD_PORT, + waitStrategy = Wait.forListeningPort))) + + protected def hiveServerJdbcUrl: String = withContainers { container => + val feHost: String = container.getServiceHost(IMPALAD_SERVICE_NAME, IMPALAD_PORT) + val fePort: Int = container.getServicePort(IMPALAD_SERVICE_NAME, IMPALAD_PORT) + s"jdbc:kyuubi://$feHost:$fePort/;auth=noSasl" + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala new file mode 100644 index 00000000000..9b31dd245bb --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/impala/WithImpalaEngine.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.jdbc.impala + +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.engine.jdbc.WithJdbcEngine + +trait WithImpalaEngine extends WithJdbcEngine with WithImpalaContainer { + + override def withKyuubiConf: Map[String, String] = Map( + ENGINE_SHARE_LEVEL.key -> "SERVER", + ENGINE_JDBC_CONNECTION_URL.key -> hiveServerJdbcUrl, + ENGINE_TYPE.key -> "jdbc", + ENGINE_JDBC_SHORT_NAME.key -> "impala", + ENGINE_JDBC_DRIVER_CLASS.key -> ImpalaConnectionProvider.driverClass) +} diff --git a/integration-tests/kyuubi-jdbc-it/pom.xml b/integration-tests/kyuubi-jdbc-it/pom.xml index 7921d94e217..5b4edbcbd62 100644 --- a/integration-tests/kyuubi-jdbc-it/pom.xml +++ b/integration-tests/kyuubi-jdbc-it/pom.xml @@ -133,6 +133,13 @@ true ${project.build.directory} + + org.apache.kyuubi + kyuubi-hive-jdbc-shaded + ${project.version} + true + ${project.build.directory} + diff --git a/integration-tests/kyuubi-jdbc-it/src/test/resources/impala-compose.yml b/integration-tests/kyuubi-jdbc-it/src/test/resources/impala-compose.yml new file mode 100644 index 00000000000..4be34042b0f --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/resources/impala-compose.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# modified compose file from Impala repo, see +# https://github.com/apache/impala/blob/4.3.0/docker/quickstart.yml +version: "3.5" +services: + metastore: + image: apache/impala:4.0.0-impala_quickstart_hms + command: ["hms"] + ports: + - "9083" + volumes: + # Volume used to store Apache Derby database. + - impala-data:/var/lib/hive + # Warehouse directory. HMS does file operations so needs access to the + # shared volume. + - impala-data:/user/hive/warehouse + - ./impala_conf:/opt/hive/conf:ro + + statestored: + image: apache/impala:4.0.0-statestored + ports: + - "25010" + command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1"] + volumes: + - ./impala_conf:/opt/impala/conf:ro + + catalogd: + depends_on: + - statestored + - metastore + image: apache/impala:4.0.0-catalogd + ports: + - "25020" + command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1", + "-hms_event_polling_interval_s=1", "-invalidate_tables_timeout_s=999999"] + volumes: + # Warehouse directory. Catalog does file operations so needs access to the + # shared volume. + - impala-data:/user/hive/warehouse + - ./impala_conf:/opt/impala/conf:ro + + impalad: + image: apache/impala:4.0.0-impalad_coord_exec + depends_on: + - statestored + - catalogd + ports: + - "21050:21050" + command: [ "-v=1", + "-redirect_stdout_stderr=false", "-logtostderr", + "-mt_dop_auto_fallback=true", + "-default_query_options=mt_dop=4,default_file_format=parquet,default_transactional_type=insert_only", + "-mem_limit=1500mb"] + environment: + # Keep the Java heap small to preserve memory for query execution. + - JAVA_TOOL_OPTIONS="-Xmx1g" + volumes: + - impala-data:/user/hive/warehouse + - ./impala_conf:/opt/impala/conf:ro + +volumes: + impala-data: + +networks: + default: + name: default-kyuubi-impala-test diff --git a/integration-tests/kyuubi-jdbc-it/src/test/resources/impala_conf/hive-site.xml b/integration-tests/kyuubi-jdbc-it/src/test/resources/impala_conf/hive-site.xml new file mode 100644 index 00000000000..55fa54ef1f8 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/resources/impala_conf/hive-site.xml @@ -0,0 +1,72 @@ + + + + + + + + hive.metastore.dml.events + true + + + + hive.metastore.event.db.notification.api.auth + false + + + hive.metastore.uris + thrift://metastore:9083 + + + + hive.metastore.warehouse.dir + /user/hive/warehouse/managed + + + hive.metastore.warehouse.external.dir + /user/hive/warehouse/external + + + + hive.support.concurrency + true + + + + hive.txn.manager + org.apache.hadoop.hive.ql.lockmgr.DbTxnManager + + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + + javax.jdo.option.ConnectionURL + jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true + + + + hive.stats.autogather + false + + diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerSuite.scala new file mode 100644 index 00000000000..8157d4d5cdd --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/OperationWithServerSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.impala.ImpalaOperationSuite + +class OperationWithServerSuite extends ImpalaOperationSuite + with WithKyuubiServerAndImpalaContainer { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerSuite.scala new file mode 100644 index 00000000000..a8e8c3f9df9 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/SessionWithServerSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.impala.SessionSuite + +class SessionWithServerSuite extends SessionSuite + with WithKyuubiServerAndImpalaContainer { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerSuite.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerSuite.scala new file mode 100644 index 00000000000..d771ef847d5 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/StatementWithServerSuite.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import org.apache.kyuubi.engine.jdbc.impala.StatementSuite + +class StatementWithServerSuite extends StatementSuite + with WithKyuubiServerAndImpalaContainer { + + override protected def jdbcUrl: String = getJdbcUrl + +} diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainer.scala new file mode 100644 index 00000000000..e835d6feac8 --- /dev/null +++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/impala/WithKyuubiServerAndImpalaContainer.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.jdbc.impala + +import java.nio.file.{Files, Path, Paths} + +import org.apache.kyuubi.{Utils, WithKyuubiServer} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME} +import org.apache.kyuubi.engine.jdbc.impala.WithImpalaEngine + +trait WithKyuubiServerAndImpalaContainer extends WithKyuubiServer with WithImpalaEngine { + + private val kyuubiHome: String = Utils + .getCodeSourceLocation(getClass).split("integration-tests").head + + private val hiveJdbcConnectorPath: String = { + val keyword = "kyuubi-hive-jdbc-shaded" + + val jarsDir = Paths.get(kyuubiHome) + .resolve("integration-tests") + .resolve("kyuubi-jdbc-it") + .resolve("target") + + Files.list(jarsDir) + .filter { p: Path => p.getFileName.toString contains keyword } + .findFirst + .orElseThrow { () => new IllegalStateException(s"Can not find $keyword in $jarsDir.") } + .toAbsolutePath + .toString + } + + override protected val conf: KyuubiConf = { + KyuubiConf() + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_JDBC_EXTRA_CLASSPATH, hiveJdbcConnectorPath) + } + + override def beforeAll(): Unit = { + val configs = withKyuubiConf + configs.foreach(config => conf.set(config._1, config._2)) + super.beforeAll() + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala index 42579fb962f..6153ac30e33 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala @@ -82,8 +82,8 @@ object KyuubiSQLException { } } - def featureNotSupported(): KyuubiSQLException = { - KyuubiSQLException("feature not supported", sqlState = "0A000") + def featureNotSupported(message: String = "feature not supported"): KyuubiSQLException = { + KyuubiSQLException(message, sqlState = "0A000") } def connectionDoesNotExist(): KyuubiSQLException = {