From 03b7613d7cc4ab4da52e316a452bbb4b818f1343 Mon Sep 17 00:00:00 2001 From: hutengfei Date: Tue, 26 Mar 2024 20:42:53 +0800 Subject: [PATCH] Array and Map not wrapped in Nullable --- .../scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala | 2 +- .../scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala | 2 +- .../scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala index 567abbff..929c7101 100644 --- a/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala +++ b/spark-3.3/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala @@ -110,7 +110,7 @@ object SchemaUtils { } private[clickhouse] def maybeNullable(chType: String, nullable: Boolean): String = - if (nullable) wrapNullable(chType) else chType + if (nullable && !chType.startsWith("Array") && !chType.startsWith("Map")) wrapNullable(chType) else chType private[clickhouse] def wrapNullable(chType: String): String = s"Nullable($chType)" } diff --git a/spark-3.4/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala b/spark-3.4/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala index 567abbff..929c7101 100644 --- a/spark-3.4/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala +++ b/spark-3.4/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala @@ -110,7 +110,7 @@ object SchemaUtils { } private[clickhouse] def maybeNullable(chType: String, nullable: Boolean): String = - if (nullable) wrapNullable(chType) else chType + if (nullable && !chType.startsWith("Array") && !chType.startsWith("Map")) wrapNullable(chType) else chType private[clickhouse] def wrapNullable(chType: String): String = s"Nullable($chType)" } diff --git a/spark-3.5/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala b/spark-3.5/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala index 567abbff..929c7101 100644 --- a/spark-3.5/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala +++ b/spark-3.5/clickhouse-spark/src/main/scala/org/apache/spark/sql/clickhouse/SchemaUtils.scala @@ -110,7 +110,7 @@ object SchemaUtils { } private[clickhouse] def maybeNullable(chType: String, nullable: Boolean): String = - if (nullable) wrapNullable(chType) else chType + if (nullable && !chType.startsWith("Array") && !chType.startsWith("Map")) wrapNullable(chType) else chType private[clickhouse] def wrapNullable(chType: String): String = s"Nullable($chType)" }