Merge branch 'dev' into improve/resource_query

.github/workflows/api-test.yml

@@ -20,6 +20,8 @@ on:
   push:
     branches:
       - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 name: API-Test
 
@@ -125,7 +127,7 @@ jobs:
         run: |
           ./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \
             -DfailIfNoTests=false \
-            -Dspotless.skip=false \
+            -Dspotless.skip=true \
             -Dtest=${{ matrix.case.class }} test
       - uses: actions/upload-artifact@v4
         if: always()

.github/workflows/codeql.yaml

@@ -23,6 +23,8 @@ on:
   pull_request:
     branches:
       - 'dev'
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 concurrency:
   group: codeql-${{ github.event.pull_request.number || github.ref }}

.github/workflows/e2e-k8s.yml

@@ -20,6 +20,8 @@ on:
   push:
     branches:
       - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 name: E2E-K8S
 

.github/workflows/e2e.yml

@@ -20,13 +20,17 @@ on:
   push:
     branches:
       - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 name: E2E
 
 concurrency:
   group: e2e-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  pull-requests: write
 
 jobs:
   paths-filter:
@@ -122,46 +126,8 @@ jobs:
             class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
           - name: ShellTaskE2ETest
             class: org.apache.dolphinscheduler.e2e.cases.tasks.ShellTaskE2ETest
-    env:
-      RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Cache local Maven repository
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
-          restore-keys: ${{ runner.os }}-maven-
-      - uses: actions/download-artifact@v4
-        name: Download Docker Images
-        with:
-          name: standalone-image
-          path: /tmp
-      - name: Load Docker Images
-        run: |
-          docker load -i /tmp/standalone-image.tar
-      - name: Run Test
-        run: |
-          ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
-            -DfailIfNoTests=false \
-            -Dtest=${{ matrix.case.class }} test
-      - uses: actions/upload-artifact@v4
-        if: always()
-        name: Upload Recording
-        with:
-          name: recording-${{ matrix.case.name }}
-          path: ${{ env.RECORDING_PATH }}
-          retention-days: 1
-  e2e-optional:
-    name: ${{ matrix.case.name }}
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        case:
+          - name: PythonTaskE2ETest
+            class: org.apache.dolphinscheduler.e2e.cases.tasks.PythonTaskE2ETest
           - name: SqlServerDataSource
             class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest
           - name: HiveDataSource
@@ -172,6 +138,15 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'adopt'
+      - name: Collect Workflow Telemetry
+        uses: ./.github/actions/workflow-telemetry-action
+        with:
+          comment_on_pr: false
       - name: Cache local Maven repository
         uses: actions/cache@v3
         with:

.github/workflows/frontend.yml

@@ -20,7 +20,8 @@ name: Frontend
 on:
   push:
     branches:
-      - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
     paths:
       - '.github/workflows/frontend.yml'
       - 'dolphinscheduler-ui/**'

.github/workflows/owasp-dependency-check.yaml

@@ -20,7 +20,8 @@ name: OWASP Dependency Check
 on:
   push:
     branches:
-      - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
   pull_request:
     paths:
       - '**/pom.xml'
@@ -30,6 +31,7 @@ env:
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@v4
         with:
@@ -47,4 +49,4 @@ jobs:
         continue-on-error: true
         with:
           name: dependency report
-          path: target/dependency-check-report.html          
+          path: target/dependency-check-report.html

.github/workflows/unit-test.yml

@@ -24,7 +24,8 @@ on:
       - '**/*.md'
       - 'dolphinscheduler-ui'
     branches:
-      - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 env:
   LOG_DIR: /tmp/dolphinscheduler

.gitmodules

@@ -25,3 +25,6 @@
 	path = .github/actions/auto-assign-action
 	url = https://github.com/kentaro-m/auto-assign-action.git
 	branch = 288f36f
+[submodule ".github/actions/workflow-telemetry-action"]
+	path = .github/actions/workflow-telemetry-action
+	url = https://github.com/catchpoint/workflow-telemetry-action

dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/ExecutorAPITest.java

@@ -31,17 +31,8 @@
 import org.apache.dolphinscheduler.common.enums.ReleaseState;
 import org.apache.dolphinscheduler.common.enums.UserType;
 import org.apache.dolphinscheduler.common.enums.WarningType;
-import org.apache.dolphinscheduler.dao.entity.Project;
 import org.apache.dolphinscheduler.dao.entity.User;
 
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Order;
-import org.junit.jupiter.api.Test;
-
-import lombok.extern.slf4j.Slf4j;
-
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.util.EntityUtils;
 
@@ -51,6 +42,14 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 
+import lombok.extern.slf4j.Slf4j;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+
 //TODO: Some test cases rely on ProcessInstance APIs. Should complete remaining cases after ProcessInstance related API tests done.
 @DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml")
 @Slf4j
@@ -80,7 +79,8 @@ public class ExecutorAPITest {
     public static void setup() {
         LoginPage loginPage = new LoginPage();
         HttpResponse loginHttpResponse = loginPage.login(username, password);
-        sessionId = JSONUtils.convertValue(loginHttpResponse.getBody().getData(), LoginResponseData.class).getSessionId();
+        sessionId =
+                JSONUtils.convertValue(loginHttpResponse.getBody().getData(), LoginResponseData.class).getSessionId();
         executorPage = new ExecutorPage(sessionId);
         processDefinitionPage = new ProcessDefinitionPage(sessionId);
         projectPage = new ProjectPage(sessionId);
@@ -103,36 +103,43 @@ public void testStartProcessInstance() {
             HttpResponse createProjectResponse = projectPage.createProject(loginUser, "project-test");
             HttpResponse queryAllProjectListResponse = projectPage.queryAllProjectList(loginUser);
             Assertions.assertTrue(queryAllProjectListResponse.getBody().getSuccess());
-            projectCode = (long) ((LinkedHashMap<String, Object>) ((List<LinkedHashMap>) queryAllProjectListResponse.getBody().getData()).get(0)).get("code");
+            projectCode = (long) ((LinkedHashMap<String, Object>) ((List<LinkedHashMap>) queryAllProjectListResponse
+                    .getBody().getData()).get(0)).get("code");
 
             // upload test workflow definition json
             ClassLoader classLoader = getClass().getClassLoader();
             File file = new File(classLoader.getResource("workflow-json/test.json").getFile());
             CloseableHttpResponse importProcessDefinitionResponse = processDefinitionPage
-                .importProcessDefinition(loginUser, projectCode, file);
+                    .importProcessDefinition(loginUser, projectCode, file);
             String data = EntityUtils.toString(importProcessDefinitionResponse.getEntity());
             Assertions.assertTrue(data.contains("\"success\":true"));
 
             // get workflow definition code
-            HttpResponse queryAllProcessDefinitionByProjectCodeResponse = processDefinitionPage.queryAllProcessDefinitionByProjectCode(loginUser, projectCode);
+            HttpResponse queryAllProcessDefinitionByProjectCodeResponse =
+                    processDefinitionPage.queryAllProcessDefinitionByProjectCode(loginUser, projectCode);
             Assertions.assertTrue(queryAllProcessDefinitionByProjectCodeResponse.getBody().getSuccess());
-            Assertions.assertTrue(queryAllProcessDefinitionByProjectCodeResponse.getBody().getData().toString().contains("hello world"));
-            processDefinitionCode = (long) ((LinkedHashMap<String, Object>) ((LinkedHashMap<String, Object>) ((List<LinkedHashMap>) queryAllProcessDefinitionByProjectCodeResponse.getBody().getData()).get(0)).get("processDefinition")).get("code");
+            Assertions.assertTrue(queryAllProcessDefinitionByProjectCodeResponse.getBody().getData().toString()
+                    .contains("hello world"));
+            processDefinitionCode =
+                    (long) ((LinkedHashMap<String, Object>) ((LinkedHashMap<String, Object>) ((List<LinkedHashMap>) queryAllProcessDefinitionByProjectCodeResponse
+                            .getBody().getData()).get(0)).get("processDefinition")).get("code");
 
             // release test workflow
-            HttpResponse releaseProcessDefinitionResponse = processDefinitionPage.releaseProcessDefinition(loginUser, projectCode, processDefinitionCode, ReleaseState.ONLINE);
+            HttpResponse releaseProcessDefinitionResponse = processDefinitionPage.releaseProcessDefinition(loginUser,
+                    projectCode, processDefinitionCode, ReleaseState.ONLINE);
             Assertions.assertTrue(releaseProcessDefinitionResponse.getBody().getSuccess());
 
             // trigger workflow instance
             SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
             Date date = new Date();
             String scheduleTime = String.format("%s,%s", formatter.format(date), formatter.format(date));
             log.info("use current time {} as scheduleTime", scheduleTime);
-            HttpResponse startProcessInstanceResponse = executorPage.startProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, FailureStrategy.END, WarningType.NONE);
+            HttpResponse startProcessInstanceResponse = executorPage.startProcessInstance(loginUser, projectCode,
+                    processDefinitionCode, scheduleTime, FailureStrategy.END, WarningType.NONE);
             Assertions.assertTrue(startProcessInstanceResponse.getBody().getSuccess());
 
             triggerCode = (long) startProcessInstanceResponse.getBody().getData();
-        }  catch (Exception e) {
+        } catch (Exception e) {
             log.error("failed", e);
             Assertions.fail();
         }
@@ -141,7 +148,8 @@ public void testStartProcessInstance() {
     @Test
     @Order(2)
     public void testStartCheckProcessDefinition() {
-        HttpResponse testStartCheckProcessDefinitionResponse = executorPage.startCheckProcessDefinition(loginUser, projectCode, processDefinitionCode);
+        HttpResponse testStartCheckProcessDefinitionResponse =
+                executorPage.startCheckProcessDefinition(loginUser, projectCode, processDefinitionCode);
         Assertions.assertTrue(testStartCheckProcessDefinitionResponse.getBody().getSuccess());
     }