Merge remote-tracking branch 'origin/Improvement-16534' into Improvem…

…ent-16534
apache · Sep 24, 2024 · 9da994c · 9da994c
2 parents c6eece1 + 85739a2
commit 9da994c
Show file tree

Hide file tree

Showing 523 changed files with 5,916 additions and 3,952 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -47,6 +47,7 @@ github:
           - Docs
           - Frontend Build
           - "E2E-K8S-Result"
+          - "API-Test-Result"
           - "Mergeable: milestone-label-check"
           - "Title Validator"
       required_pull_request_reviews:

diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml
@@ -51,6 +51,19 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Maximize runner space
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 30720
+          temp-reserve-mb: 10240
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - uses: actions/checkout@v4
         with:
           submodules: true
@@ -62,13 +75,13 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
           ./mvnw -B clean install \
-          -Dmaven.test.skip \
-          -Dmaven.javadoc.skip \
+          -Dmaven.test.skip=true \
+          -Dmaven.javadoc.skip=true \
           -Dspotless.skip=true \
           -Pdocker,release -Ddocker.tag=ci
       - name: Export Docker Images
@@ -122,7 +135,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images

diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -56,6 +56,19 @@ jobs:
         java: [ '8', '11' ]
     timeout-minutes: 30
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Maximize runner space
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 30720
+          temp-reserve-mb: 10240
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - uses: actions/checkout@v4
         with:
           submodules: true
@@ -78,10 +91,7 @@ jobs:
           ./mvnw -B clean install \
                  -Prelease \
                  -Dmaven.test.skip=true \
-                 -Dspotless.skip=true \
-                 -Dhttp.keepAlive=false \
-                 -Dmaven.wagon.http.pool=false \
-                 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
+                 -Dspotless.skip=true
       - name: Check dependency license
         run: tools/dependencies/check-LICENSE.sh
       - uses: actions/upload-artifact@v4
@@ -124,39 +134,9 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
-  schema-check-prepare:
-    name: schema-check-prepare-${{ matrix.version }}
-    needs: paths-filter
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        version: [ "3.1.9", "3.2.0" ]
-    steps:
-      - uses: actions/cache/restore@v4
-        name: Download Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
-          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
-      - name: Download Binary Package if not cached
-        run: |
-          if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then
-            echo "Binary package not found in cache, downloading..."
-            mkdir -p /tmp/${{ matrix.version }}
-            wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }}
-          else
-            echo "Binary package found in cache, skipping download."
-          fi
-      - uses: actions/cache/save@v4
-        name: Upload Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
   schema-check:
     name: ${{ matrix.case.name }}-${{ matrix.version }}
-    needs: [ build, schema-check-prepare]
+    needs: [ build ]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -181,12 +161,6 @@ jobs:
         with:
           name: binary-package-8
           path: ds_schema_check_test/dev
-      - uses: actions/cache/restore@v4
-        name: Download Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
-          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
       - name: Running Schema Check
         run: |
           /bin/bash ${{ matrix.case.script }} ${{ matrix.version }}

diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile
@@ -22,22 +22,24 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
 
 #Download mysql jar
-ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
-ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
-RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
+ENV MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+ENV MYSQL_DRIVER="mysql-connector-java-8.0.16.jar"
+RUN wget -O $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $MYSQL_URL
+RUN cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
 
 #Deploy
 COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh

diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh
diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile
@@ -22,22 +22,24 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
 
 #Download mysql jar
-ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
-ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
-RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
+ENV MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+ENV MYSQL_DRIVER="mysql-connector-java-8.0.16.jar"
+RUN wget -O $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $MYSQL_URL
+RUN cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
 
 #Deploy
 COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh

diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
@@ -22,10 +22,11 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile
@@ -22,10 +22,11 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh