Spark 3.3: Test with ClickHouse Cloud

ClickHouse · Jul 24, 2024 · 276d094 · 276d094
1 parent acdc4f7
commit 276d094
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 15 deletions.
diff --git a/spark-3.3/build.gradle b/spark-3.3/build.gradle
@@ -90,4 +90,8 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") {
     slowTest {
         classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf")
     }
+
+    cloudTest {
+        classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf")
+    }
 }
diff --git a/...rk-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala b/...rk-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala
@@ -14,15 +14,22 @@
 
 package org.apache.spark.sql.clickhouse.single
 
+import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn}
 import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.USE_NULLABLE_QUERY_SCHEMA
 import org.apache.spark.sql.clickhouse.SparkUtils
 import org.apache.spark.sql.types.DataTypes.{createArrayType, createMapType}
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.{DataFrame, Row}
+import org.scalatest.tags.Cloud
 
 import java.math.MathContext
 
-class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest {
+@Cloud
+class ClickHouseCloudDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn
+
+class ClickHouseSingleDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn
+
+abstract class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest {
 
   val SPARK_43390_ENABLED: Boolean = sys.env.contains("SPARK_43390_ENABLED") || {
     SparkUtils.MAJOR_MINOR_VERSION match {

diff --git a/...khouse/single/ClickHouseSingleSuite.scala → ...house/single/ClickHouseGenericSuite.scala b/...khouse/single/ClickHouseSingleSuite.scala → ...house/single/ClickHouseGenericSuite.scala
@@ -14,12 +14,19 @@
 
 package org.apache.spark.sql.clickhouse.single
 
+import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.types._
+import org.scalatest.tags.Cloud
 
-class ClickHouseSingleSuite extends SparkClickHouseSingleTest {
+@Cloud
+class ClickHouseCloudGenericSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn
+
+class ClickHouseSingleGenericSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn
+
+abstract class ClickHouseGenericSuite extends SparkClickHouseSingleTest {
 
   import testImplicits._
 

diff --git a/...rk-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala b/...rk-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala
@@ -14,9 +14,16 @@
 
 package org.apache.spark.sql.clickhouse.single
 
+import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn}
 import org.apache.spark.sql.Row
+import org.scalatest.tags.Cloud
 
-class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest {
+@Cloud
+class ClickHouseCloudTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseCloudMixIn
+
+class ClickHouseSingleTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseSingleMixIn
+
+abstract class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest {
 
   import testImplicits._
 

diff --git a/...-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala b/...-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala
@@ -14,13 +14,13 @@
 
 package org.apache.spark.sql.clickhouse.single
 
+import com.clickhouse.spark.base.ClickHouseProvider
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.clickhouse.SparkTest
 import org.apache.spark.sql.functions.month
 import org.apache.spark.sql.types.StructType
-import com.clickhouse.spark.base.ClickHouseSingleMixIn
 
-trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn {
+trait SparkClickHouseSingleTest extends SparkTest with ClickHouseProvider {
 
   import testImplicits._
 
@@ -34,10 +34,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn {
     .set("spark.sql.catalog.clickhouse.host", clickhouseHost)
     .set("spark.sql.catalog.clickhouse.http_port", clickhouseHttpPort.toString)
     .set("spark.sql.catalog.clickhouse.protocol", "http")
-    .set("spark.sql.catalog.clickhouse.user", CLICKHOUSE_USER)
-    .set("spark.sql.catalog.clickhouse.password", CLICKHOUSE_PASSWORD)
-    .set("spark.sql.catalog.clickhouse.database", CLICKHOUSE_DB)
+    .set("spark.sql.catalog.clickhouse.user", clickhouseUser)
+    .set("spark.sql.catalog.clickhouse.password", clickhousePassword)
+    .set("spark.sql.catalog.clickhouse.database", clickhouseDatabase)
     .set("spark.sql.catalog.clickhouse.option.custom_http_params", "async_insert=1,wait_for_async_insert=1")
+    .set("spark.sql.catalog.clickhouse.option.ssl", isSslEnabled.toString)
     // extended configurations
     .set("spark.clickhouse.write.batchSize", "2")
     .set("spark.clickhouse.write.maxRetry", "2")
@@ -51,9 +52,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn {
     "host" -> clickhouseHost,
     "http_port" -> clickhouseHttpPort.toString,
     "protocol" -> "http",
-    "user" -> CLICKHOUSE_USER,
-    "password" -> CLICKHOUSE_PASSWORD,
-    "database" -> CLICKHOUSE_DB
+    "user" -> clickhouseUser,
+    "password" -> clickhousePassword,
+    "database" -> clickhouseDatabase,
+    "option.custom_http_params" -> "async_insert=1,wait_for_async_insert=1",
+    "option.ssl" -> isSslEnabled.toString
   )
 
   def withTable(

diff --git a/...lickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala b/...lickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala
@@ -14,12 +14,18 @@
 
 package org.apache.spark.sql.clickhouse.single
 
+import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.clickhouse.TPCDSTestUtils
-import org.scalatest.tags.Slow
+import org.scalatest.tags.{Cloud, Slow}
+
+@Cloud
+class ClickHouseCloudTPCDSSuite extends TPCDSSuite with ClickHouseCloudMixIn
 
 @Slow
-class TPCDSSuite extends SparkClickHouseSingleTest {
+class ClickHouseSingleTPCDSSuite extends TPCDSSuite with ClickHouseSingleMixIn
+
+abstract class TPCDSSuite extends SparkClickHouseSingleTest {
 
   override protected def sparkConf: SparkConf = super.sparkConf
     .set("spark.sql.catalog.tpcds", "org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog")

diff --git a/...test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala b/...test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala
@@ -14,14 +14,21 @@
 
 package org.apache.spark.sql.clickhouse.single
 
+import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn}
 import org.apache.spark.sql.clickhouse.ClickHouseSQLConf._
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.StringType
 import org.apache.spark.sql.{AnalysisException, Row}
+import org.scalatest.tags.Cloud
 
-import java.time.LocalDate
+@Cloud
+class ClickHouseCloudsWriteDistributionAndOrderingSuite
+    extends WriteDistributionAndOrderingSuite with ClickHouseCloudMixIn
 
-class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest {
+class ClickHouseSinglesWriteDistributionAndOrderingSuite
+    extends WriteDistributionAndOrderingSuite with ClickHouseSingleMixIn
+
+abstract class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest {
 
   import testImplicits._