Merge branch 'apache:master' into master

davidyuan1223 · Jan 26, 2024 · 46001d4 · 46001d4
2 parents 6376466 + 47a1091
commit 46001d4
Show file tree

Hide file tree

Showing 143 changed files with 3,322 additions and 1,665 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
@@ -29,28 +29,10 @@ Please include a summary of the change and which issue is fixed. Please also inc
 
 ---
 
-# Checklists
-## 📝 Author Self Checklist
+# Checklist 📝
 <!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
 <!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-- [ ] My code follows the [style guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html) of this project
-- [ ] I have performed a self-review
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] I have made corresponding changes to the documentation
-- [ ] My changes generate no new warnings
-- [ ] I have added tests that prove my fix is effective or that my feature works
-- [ ] New and existing unit tests pass locally with my changes
-- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)
-
-## 📝 Committer Pre-Merge Checklist
-
-- [ ] Pull request title is okay.
-- [ ] No license issues.
-- [ ] Milestone correctly set?
-- [ ] Test coverage is ok
-- [ ] Assignees are selected.
-- [ ] Minimum number of approvals
-- [ ] No changes are requested
 
+- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)
 
 **Be nice. Be informative.**
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -129,8 +129,7 @@
       '.dockerignore',
       'bin/docker-image-tool.sh',
       'docker/**/*',
-      'integration-tests/kyuubi-kubernetes-it/**/*',
-      'tools/spark-block-cleaner/**/*'
+      'integration-tests/kyuubi-kubernetes-it/**/*'
     ]
 
 "module:metrics":
@@ -164,8 +163,7 @@
   - changed-files:
       - any-glob-to-any-file: [
         'externals/kyuubi-spark-sql-engine/**/*',
-        'extensions/spark/**/*',
-        'tools/spark-block-cleaner/**/*'
+        'extensions/spark/**/*'
       ]
 
 "module:extensions":

diff --git a/.github/workflows/gluten.yml b/.github/workflows/gluten.yml
@@ -99,11 +99,6 @@ jobs:
         with:
           path: gluten/package/target/
           key: gluten_package_${{ steps.date.outputs.date }}
-      - name: Cache Gluten Package
-        uses: actions/cache@v3
-        with:
-          path: gluten/package/target/
-          key: gluten_package
       - name: Setup JDK 8
         uses: actions/setup-java@v4
         with:

diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
@@ -44,7 +44,7 @@ jobs:
           check-latest: false
       - run: >-
           build/mvn org.apache.rat:apache-rat-plugin:check
-          -Ptpcds -Pspark-block-cleaner -Pkubernetes-it
+          -Ptpcds -Pkubernetes-it
           -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 -Pspark-3.4 -Pspark-3.5
       - name: Upload rat report
         if: failure()

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -356,9 +356,9 @@ jobs:
         uses: ./.github/actions/cache-engine-archives
       - name: Build and test Trino with maven w/o linters
         run: |
-          TEST_MODULES="kyuubi-server,externals/kyuubi-trino-engine,externals/kyuubi-spark-sql-engine,externals/kyuubi-download,integration-tests/kyuubi-trino-it"
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am -Pflink-provided -Phive-provided clean install -DskipTests
-          ./build/mvn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -pl ${TEST_MODULES} -am -Pflink-provided -Phive-provided test -Dtest=none -DwildcardSuites=org.apache.kyuubi.it.trino.operation.TrinoOperationSuite,org.apache.kyuubi.it.trino.server.TrinoFrontendSuite
+          TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it"
+          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
+          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         profiles:
-          - '-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.5,spark-3.4,spark-3.3,spark-3.2,tpcds,kubernetes-it'
+          - '-Pflink-provided,hive-provided,spark-provided,spark-3.5,spark-3.4,spark-3.3,spark-3.2,tpcds,kubernetes-it'
 
     steps:
       - uses: actions/checkout@v4
@@ -65,7 +65,7 @@ jobs:
         if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure'
         run: |
           MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip"
-          build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds
+          build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-3.2,tpcds
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3
           build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4

diff --git a/.gitignore b/.gitignore
@@ -80,3 +80,7 @@ conf/kyuubi-env.sh
 kyuubi-server/gen
 kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/*.tokens
 kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/gen/
+
+# For draw.io
+.$*.bkp
+.$*.dtmp
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -214,7 +214,7 @@ commons-logging:commons-logging
 org.apache.commons:commons-lang3
 org.apache.derby:derby
 com.google.errorprone:error_prone_annotations
-net.jodah:failsafe
+dev.failsafe:failsafe
 com.jakewharton.fliptables:fliptables
 com.github.mifmif:generex
 io.grpc:grpc-api
@@ -225,6 +225,7 @@ io.grpc:grpc-netty
 io.grpc:grpc-protobuf-lite
 io.grpc:grpc-protobuf
 io.grpc:grpc-stub
+io.grpc:grpc-util
 com.google.code.gson:gson
 com.google.guava:failureaccess
 com.google.guava:guava

diff --git a/NOTICE b/NOTICE
@@ -1,7 +1,35 @@
 
 Apache Kyuubi
-Copyright 2021-2023 The Apache Software Foundation.
+Copyright 2021-2024 The Apache Software Foundation.
 
 This product includes software developed at
 The Apache Software Foundation (https://www.apache.org/).
 
+Apache Hive
+Copyright 2008-2019 The Apache Software Foundation
+
+Apache Iceberg
+Copyright 2017-2022 The Apache Software Foundation
+
+Apache Parquet MR
+Copyright 2014-2024 The Apache Software Foundation
+
+This project includes code from Kite, developed at Cloudera, Inc. with
+the following copyright notice:
+
+| Copyright 2013 Cloudera Inc.
+|
+| Licensed under the Apache License, Version 2.0 (the "License");
+| you may not use this file except in compliance with the License.
+| You may obtain a copy of the License at
+|
+|   http://www.apache.org/licenses/LICENSE-2.0
+|
+| Unless required by applicable law or agreed to in writing, software
+| distributed under the License is distributed on an "AS IS" BASIS,
+| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+| See the License for the specific language governing permissions and
+| limitations under the License.
+
+Apache Spark
+Copyright 2014 and onwards The Apache Software Foundation.
diff --git a/NOTICE-binary b/NOTICE-binary
@@ -1,10 +1,36 @@
 
 Apache Kyuubi
-Copyright 2021-2023 The Apache Software Foundation.
+Copyright 2021-2024 The Apache Software Foundation.
 
 This product includes software developed at
 The Apache Software Foundation (https://www.apache.org/).
 
+Apache Iceberg
+Copyright 2017-2022 The Apache Software Foundation
+
+Apache Parquet MR
+Copyright 2014-2024 The Apache Software Foundation
+
+This project includes code from Kite, developed at Cloudera, Inc. with
+the following copyright notice:
+
+| Copyright 2013 Cloudera Inc.
+|
+| Licensed under the Apache License, Version 2.0 (the "License");
+| you may not use this file except in compliance with the License.
+| You may obtain a copy of the License at
+|
+|   http://www.apache.org/licenses/LICENSE-2.0
+|
+| Unless required by applicable law or agreed to in writing, software
+| distributed under the License is distributed on an "AS IS" BASIS,
+| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+| See the License for the specific language governing permissions and
+| limitations under the License.
+
+Apache Spark
+Copyright 2014 and onwards The Apache Software Foundation.
+
 # Notices for Eclipse GlassFish
 
 This content is produced and maintained by the Eclipse GlassFish project.

diff --git a/bin/beeline b/bin/beeline
@@ -19,7 +19,9 @@
 ## Kyuubi BeeLine Entrance
 CLASS="org.apache.hive.beeline.KyuubiBeeLine"
 
-export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
+if [ -z "${KYUUBI_HOME}" ]; then
+  KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
 
 . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" -s
 

diff --git a/bin/kyuubi b/bin/kyuubi
@@ -62,7 +62,9 @@ function kyuubi_rotate_log() {
   fi
 }
 
-export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
+if [ -z "${KYUUBI_HOME}" ]; then
+  KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
 
 if [[ $1 == "start" ]] || [[ $1 == "run" ]]; then
   . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh"

diff --git a/bin/kyuubi-admin b/bin/kyuubi-admin
@@ -19,7 +19,9 @@
 ## Kyuubi Admin Control Client Entrance
 CLASS="org.apache.kyuubi.ctl.cli.AdminControlCli"
 
-export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
+if [ -z "${KYUUBI_HOME}" ]; then
+  KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
 
 . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" -s
 

diff --git a/bin/kyuubi-ctl b/bin/kyuubi-ctl
@@ -19,7 +19,9 @@
 ## Kyuubi Control Client Entrance
 CLASS="org.apache.kyuubi.ctl.cli.ControlCli"
 
-export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
+if [ -z "${KYUUBI_HOME}" ]; then
+  KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
 
 . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" -s
 

diff --git a/bin/kyuubi-zk-cli b/bin/kyuubi-zk-cli
@@ -19,8 +19,9 @@
 ## Zookeeper Shell Client Entrance
 CLASS="org.apache.kyuubi.shaded.zookeeper.ZooKeeperMain"
 
-export KYUUBI_HOME="$(cd "$(dirname "$0")"/..; pwd)"
-
+if [ -z "${KYUUBI_HOME}" ]; then
+  KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
 . "${KYUUBI_HOME}/bin/load-kyuubi-env.sh" -s
 
 if [[ -z ${JAVA_HOME} ]]; then

diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh
@@ -16,9 +16,7 @@
 # limitations under the License.
 #
 
-
 export KYUUBI_HOME="${KYUUBI_HOME:-"$(cd "$(dirname "$0")"/.. || exit; pwd)"}"
-
 export KYUUBI_CONF_DIR="${KYUUBI_CONF_DIR:-"${KYUUBI_HOME}"/conf}"
 
 silent=0

diff --git a/build/dist b/build/dist
@@ -330,14 +330,6 @@ for jar in $(ls "$DISTDIR/jars/"); do
   fi
 done
 
-# Copy kyuubi tools
-if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then
-  mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes"
-  mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars"
-  cp -r "$KYUUBI_HOME"/tools/spark-block-cleaner/kubernetes/* "$DISTDIR/tools/spark-block-cleaner/kubernetes/"
-  cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/tools/spark-block-cleaner/jars/"
-fi
-
 # Copy Kyuubi Spark extension
 SPARK_EXTENSION_VERSIONS=('3-1' '3-2' '3-3' '3-4' '3-5')
 # shellcheck disable=SC2068

diff --git a/charts/kyuubi/templates/kyuubi-configmap.yaml b/charts/kyuubi/templates/kyuubi-configmap.yaml
@@ -24,7 +24,6 @@ metadata:
 data:
   {{- with .Values.kyuubiConf.kyuubiEnv }}
   kyuubi-env.sh: |
-    #!/usr/bin/env bash
     {{- tpl . $ | nindent 4 }}
   {{- end }}
   kyuubi-defaults.conf: |

diff --git a/charts/kyuubi/templates/kyuubi-spark-configmap.yaml b/charts/kyuubi/templates/kyuubi-spark-configmap.yaml
@@ -0,0 +1,40 @@
+{{/*
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/}}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-spark
+  labels:
+    {{- include "kyuubi.labels" . | nindent 4 }}
+data:
+  {{- with .Values.sparkConf.sparkEnv }}
+  spark-env.sh: |
+    {{- tpl . $ | nindent 4 }}
+  {{- end }}
+  {{- with .Values.sparkConf.sparkDefaults }}
+  spark-defaults.conf: |
+    {{- tpl . $ | nindent 4 }}
+  {{- end }}
+  {{- with .Values.sparkConf.log4j2 }}
+  log4j2.properties: |
+    {{- tpl . $ | nindent 4 }}
+  {{- end }}
+  {{- with .Values.sparkConf.metrics }}
+  metrics.properties: |
+    {{- tpl . $ | nindent 4 }}
+  {{- end }}
diff --git a/charts/kyuubi/templates/kyuubi-statefulset.yaml b/charts/kyuubi/templates/kyuubi-statefulset.yaml
@@ -62,9 +62,14 @@ spec:
           {{- with .Values.args }}
           args: {{- tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
-          {{- with .Values.env }}
-          env: {{- tpl (toYaml .) $ | nindent 12 }}
-          {{- end }}
+          env:
+            - name: KYUUBI_CONF_DIR
+              value: {{ .Values.kyuubiConfDir }}
+            - name: SPARK_CONF_DIR
+              value: {{ .Values.sparkConfDir }}
+            {{- with .Values.env }}
+              {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
           {{- with .Values.envFrom }}
           envFrom: {{- tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
@@ -105,6 +110,8 @@ spec:
           volumeMounts:
             - name: conf
               mountPath: {{ .Values.kyuubiConfDir }}
+            - name: conf-spark
+              mountPath: {{ .Values.sparkConfDir }}
             {{- with .Values.volumeMounts }}
               {{- tpl (toYaml .) $ | nindent 12 }}
             {{- end }}
@@ -115,6 +122,9 @@ spec:
         - name: conf
           configMap:
             name: {{ .Release.Name }}
+        - name: conf-spark
+          configMap:
+            name: {{ .Release.Name }}-spark
         {{- with .Values.volumes }}
           {{- tpl (toYaml .) $ | nindent 8 }}
         {{- end }}