[KYUUBI #5738][FEATURE] Add a spark commit protocol to support compact small file before commit job

extensions/spark/kyuubi-extension-spark-compaction/README.md

@@ -0,0 +1,34 @@
+<!--
+- Licensed to the Apache Software Foundation (ASF) under one or more
+- contributor license agreements.  See the NOTICE file distributed with
+- this work for additional information regarding copyright ownership.
+- The ASF licenses this file to You under the Apache License, Version 2.0
+- (the "License"); you may not use this file except in compliance with
+- the License.  You may obtain a copy of the License at
+-
+-   http://www.apache.org/licenses/LICENSE-2.0
+-
+- Unless required by applicable law or agreed to in writing, software
+- distributed under the License is distributed on an "AS IS" BASIS,
+- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+- See the License for the specific language governing permissions and
+- limitations under the License.
+-->
+
+# Kyuubi Spark Compact Commit Protocol
+
+## Functions
+
+- [x] Enable compact small file before commit job
+
+## Build
+
+```shell
+build/mvn clean package -DskipTests -pl :kyuubi-extention-spark-compaction_2.12 -am -Dspark.version=3.2.1
+```
+
+## Use
+
+```text
+spark.sql.sources.commitProtocolClass=org.apache.spark.sql.execution.CompactFilesCommitProtocol
+```

extensions/spark/kyuubi-extension-spark-compaction/pom.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.kyuubi</groupId>
+        <artifactId>kyuubi-parent</artifactId>
+        <version>1.9.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+
+    </parent>
+
+    <artifactId>kyuubi-extension-spark-compaction_${scala.binary.version}</artifactId>
+    <packaging>jar</packaging>
+    <name>Kyuubi Spark JDBC Dialect plugin</name>
+    <url>https://kyuubi.apache.org/</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+
+        <testResources>
+            <testResource>
+                <directory>${project.basedir}/src/test/resources</directory>
+            </testResource>
+        </testResources>
+        <outputDirectory>target/scala-${scala.binary.verison}/classes</outputDirectory>
+        <testOutputDirectory>target/scala-${scala.binary.verison}/test-classes</testOutputDirectory>
+    </build>
+
+</project>

extensions/spark/kyuubi-extension-spark-compaction/src/main/scala/org/apache/spark/sql/execution/CompactConf.scala

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.internal.SQLConf.buildConf
+
+object CompactConf {
+  val COMPACT_PREPARE_PARALLELISM =
+    buildConf("spark.compact.prepare.parallelism")
+      .doc("The compute parallelism during group all committed task file.")
+      .version("1.9.0")
+      .intConf
+      .createWithDefault(5)
+
+  val COMPACT_TARGET_SIZE =
+    buildConf("spark.compact.targetFileSize")
+      .doc("")
+      .version("1.9.0")
+      .longConf
+      .createWithDefault(128 * 1024 * 1024) // 128m
+
+  val COMPACT_FILE_THRESHOLD =
+    buildConf("spark.compact.fileThreshold")
+      .doc("")
+      .version("1.9.0")
+      .longConf
+      .createWithDefault(32 * 1024 * 1024) // 32m
+}