diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala index b6881019..66efca97 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.USE_NULLABLE_QUERY_SCHEMA import org.apache.spark.sql.clickhouse.SparkUtils import org.apache.spark.sql.types.DataTypes.{createArrayType, createMapType} import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row} +import org.scalatest.tags.Cloud import java.math.MathContext -class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { val SPARK_43390_ENABLED: Boolean = sys.env.contains("SPARK_43390_ENABLED") || { SparkUtils.MAJOR_MINOR_VERSION match { diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala similarity index 97% rename from spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala rename to spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala index b80715ee..7d10753d 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala @@ -14,12 +14,19 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.types._ +import org.scalatest.tags.Cloud -class ClickHouseSingleSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudGenericSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleGenericSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseGenericSuite extends SparkClickHouseSingleTest { import testImplicits._ diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala index 7409a590..2879ae47 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala @@ -14,9 +14,16 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseCloudMixIn + +class ClickHouseSingleTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseSingleMixIn + +abstract class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { import testImplicits._ diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala index 65f7f372..d9e7890a 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala @@ -14,13 +14,13 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.ClickHouseProvider import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.SparkTest import org.apache.spark.sql.functions.month import org.apache.spark.sql.types.StructType -import com.clickhouse.spark.base.ClickHouseSingleMixIn -trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { +trait SparkClickHouseSingleTest extends SparkTest with ClickHouseProvider { import testImplicits._ @@ -34,10 +34,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { .set("spark.sql.catalog.clickhouse.host", clickhouseHost) .set("spark.sql.catalog.clickhouse.http_port", clickhouseHttpPort.toString) .set("spark.sql.catalog.clickhouse.protocol", "http") - .set("spark.sql.catalog.clickhouse.user", CLICKHOUSE_USER) - .set("spark.sql.catalog.clickhouse.password", CLICKHOUSE_PASSWORD) - .set("spark.sql.catalog.clickhouse.database", CLICKHOUSE_DB) + .set("spark.sql.catalog.clickhouse.user", clickhouseUser) + .set("spark.sql.catalog.clickhouse.password", clickhousePassword) + .set("spark.sql.catalog.clickhouse.database", clickhouseDatabase) .set("spark.sql.catalog.clickhouse.option.custom_http_params", "async_insert=1,wait_for_async_insert=1") + .set("spark.sql.catalog.clickhouse.option.ssl", isSslEnabled.toString) // extended configurations .set("spark.clickhouse.write.batchSize", "2") .set("spark.clickhouse.write.maxRetry", "2") @@ -51,9 +52,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { "host" -> clickhouseHost, "http_port" -> clickhouseHttpPort.toString, "protocol" -> "http", - "user" -> CLICKHOUSE_USER, - "password" -> CLICKHOUSE_PASSWORD, - "database" -> CLICKHOUSE_DB + "user" -> clickhouseUser, + "password" -> clickhousePassword, + "database" -> clickhouseDatabase, + "option.custom_http_params" -> "async_insert=1,wait_for_async_insert=1", + "option.ssl" -> isSslEnabled.toString ) def withTable( diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala index cda9793e..7854d8de 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala @@ -14,12 +14,19 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.TPCDSTestUtils -import org.scalatest.tags.Slow +import org.scalatest.tags.{Cloud, Slow} @Slow -class TPCDSSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudTPCDSSuite extends TPCDSSuite with ClickHouseCloudMixIn + +@Slow +class ClickHouseSingleTPCDSSuite extends TPCDSSuite with ClickHouseSingleMixIn + +abstract class TPCDSSuite extends SparkClickHouseSingleTest { override protected def sparkConf: SparkConf = super.sparkConf .set("spark.sql.catalog.tpcds", "org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog") diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala index fe9ba535..e38435ba 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala @@ -14,14 +14,21 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType -import org.apache.spark.sql.{AnalysisException, Row} +import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -import java.time.LocalDate +@Cloud +class ClickHouseCloudsWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseCloudMixIn -class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { +class ClickHouseSinglesWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseSingleMixIn + +abstract class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { import testImplicits._