3.3 done waiting for testing

ClickHouse · Jun 17, 2024 · afe8e56 · afe8e56
1 parent 89b0c90
commit afe8e56
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 3 deletions.
diff --git a/...3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ClickHouseSQLConf.scala b/...3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ClickHouseSQLConf.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit
 
 /**
  * Run the following command to update the configuration docs.
- *   UPDATE=1 ./gradlew test --tests=ConfigurationSuite
+ * UPDATE=1 ./gradlew test --tests=ConfigurationSuite
  */
 object ClickHouseSQLConf {
 
@@ -201,4 +201,12 @@ object ClickHouseSQLConf {
       .version("0.8.0")
       .booleanConf
       .createWithDefault(false)
+
+  val FIXED_STRING_READ_AS: ConfigEntry[String] =
+    buildConf("spark.clickhouse.fixedStringReadAs")
+      .doc("read ClickHouse FixedString type as the specified Spark data type. Supported formats: binary, string")
+      .version("0.8.1")
+      .stringConf
+      .transform(_.toLowerCase)
+      .createWithDefault("binary")
 }
diff --git a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala
@@ -18,15 +18,22 @@ import com.clickhouse.data.ClickHouseDataType._
 import com.clickhouse.data.{ClickHouseColumn, ClickHouseDataType}
 import org.apache.spark.sql.types._
 import xenon.clickhouse.exception.CHClientException
+import org.apache.spark.sql.catalyst.SQLConfHelper
+import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.FIXED_STRING_READ_AS
 
-object SchemaUtils {
+object SchemaUtils extends SQLConfHelper {
 
   def fromClickHouseType(chColumn: ClickHouseColumn): (DataType, Boolean) = {
     val catalystType = chColumn.getDataType match {
       case Nothing => NullType
       case Bool => BooleanType
       case String | JSON | UUID | Enum8 | Enum16 | IPv4 | IPv6 => StringType
-      case FixedString => BinaryType
+      case FixedString =>
+        conf.getConf(FIXED_STRING_READ_AS) match {
+          case "binary" => BinaryType
+          case "string" => StringType
+          case unsupported => throw CHClientException(s"Unsupported fixed string read format mapping: $unsupported")
+        }
       case Int8 => ByteType
       case UInt8 | Int16 => ShortType
       case UInt16 | Int32 => IntegerType