diff --git a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ClickHouseSQLConf.scala b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ClickHouseSQLConf.scala index c61218af..82c07cc9 100644 --- a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ClickHouseSQLConf.scala +++ b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/ClickHouseSQLConf.scala @@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit /** * Run the following command to update the configuration docs. - * UPDATE=1 ./gradlew test --tests=ConfigurationSuite + * UPDATE=1 ./gradlew test --tests=ConfigurationSuite */ object ClickHouseSQLConf { @@ -201,4 +201,12 @@ object ClickHouseSQLConf { .version("0.8.0") .booleanConf .createWithDefault(false) + + val FIXED_STRING_READ_AS: ConfigEntry[String] = + buildConf("spark.clickhouse.fixedStringReadAs") + .doc("read ClickHouse FixedString type as the specified Spark data type. Supported formats: binary, string") + .version("0.8.1") + .stringConf + .transform(_.toLowerCase) + .createWithDefault("binary") } diff --git a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala index 54d88c75..d72f885f 100644 --- a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala +++ b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala @@ -18,15 +18,22 @@ import com.clickhouse.data.ClickHouseDataType._ import com.clickhouse.data.{ClickHouseColumn, ClickHouseDataType} import org.apache.spark.sql.types._ import xenon.clickhouse.exception.CHClientException +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.FIXED_STRING_READ_AS -object SchemaUtils { +object SchemaUtils extends SQLConfHelper { def fromClickHouseType(chColumn: ClickHouseColumn): (DataType, Boolean) = { val catalystType = chColumn.getDataType match { case Nothing => NullType case Bool => BooleanType case String | JSON | UUID | Enum8 | Enum16 | IPv4 | IPv6 => StringType - case FixedString => BinaryType + case FixedString => + conf.getConf(FIXED_STRING_READ_AS) match { + case "binary" => BinaryType + case "string" => StringType + case unsupported => throw CHClientException(s"Unsupported fixed string read format mapping: $unsupported") + } case Int8 => ByteType case UInt8 | Int16 => ShortType case UInt16 | Int32 => IntegerType