apache · ruanwenjun · Jun 28, 2024 · Jun 26, 2024 · Jun 26, 2024 · Jun 26, 2024
@@ -145,10 +145,6 @@ export default {
                                 title: 'Sqoop',
                                 link: '/en-us/docs/dev/user_doc/guide/task/sqoop.html',
                             },
-                            {
-                                title: 'Pigeon',
-                                link: '/en-us/docs/dev/user_doc/guide/task/pigeon.html',
-                            },
                             {
                                 title: 'Conditions',
                                 link: '/en-us/docs/dev/user_doc/guide/task/conditions.html',
@@ -877,10 +873,6 @@ export default {
                                 title: 'Sqoop',
                                 link: '/zh-cn/docs/dev/user_doc/guide/task/sqoop.html',
                             },
-                            {
-                                title: 'Pigeon',
-                                link: '/zh-cn/docs/dev/user_doc/guide/task/pigeon.html',
-                            },
                             {
                                 title: 'Conditions',
                                 link: '/zh-cn/docs/dev/user_doc/guide/task/conditions.html',

@@ -29,4 +29,5 @@ This document records the incompatible updates between each version. You need to
 ## 3.3.0
 
 * Remove the `udf-manage` function from the `resource center` ([#16209])
+* Remove the `Pigeon` from the `Task Plugin` ([#16218])
 
@@ -27,4 +27,5 @@
 ## 3.3.0
 
 * 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209])
+* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218])
 
@@ -28,5 +28,3 @@ dynamic-task:
     - {name: DATA_QUALITY,icon: shell-icon.png,hover: shell-hover.png}
   machineLearning:
     - {name: JUPYTER,icon: shell-icon.png,hover: shell-hover.png}
-  other:
-    - {name: PIGEON,icon: shell-icon.png,hover: shell-hover.png}
@@ -56,7 +56,6 @@ task:
     - 'PYTORCH'
     - 'KUBEFLOW'
   other:
-    - 'PIGEON'
     - 'ZEPPELIN'
     - 'CHUNJUN'
     - 'DATASYNC'

@@ -31,7 +31,6 @@
         <netty.version>4.1.53.Final</netty.version>
         <spring-boot.version>2.7.3</spring-boot.version>
         <spring-ldap.version>2.4.1</spring-ldap.version>
-        <java-websocket.version>1.5.1</java-websocket.version>
         <mybatis-plus.version>3.5.2</mybatis-plus.version>
         <quartz.version>2.3.2</quartz.version>
         <druid.version>1.2.20</druid.version>
@@ -169,11 +168,6 @@
                 <scope>test</scope>
             </dependency>
 
-            <dependency>
-                <groupId>org.java-websocket</groupId>
-                <artifactId>Java-WebSocket</artifactId>
-                <version>${java-websocket.version}</version>
-            </dependency>
             <!-- mybatis -->
             <dependency>
                 <groupId>com.baomidou</groupId>

@@ -26,7 +26,7 @@ data.basedir.path=/tmp/dolphinscheduler
 # use shared file mount point
 resource.storage.type=LOCAL
 # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
-resource.storage.upload.base.path=/dolphinscheduler
+resource.storage.upload.base.path=/tmp/dolphinscheduler
 
 # The Azure client ID (Azure Application (client) ID)
 resource.azure.client.id=minioadmin

@@ -666,7 +666,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     animal-sniffer-annotations 1.19 https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.19, MIT
     checker-qual 3.12.0 https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.12.0, MIT + GPLv2
     checker-qual 3.19.0 https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.19.0, MIT + GPLv2
-    Java-WebSocket 1.5.1: https://github.com/TooTallNate/Java-WebSocket,  MIT
     oshi-core 6.1.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/6.1.1, MIT
     unirest-java 3.7.04-standalone: https://mvnrepository.com/artifact/com.konghq/unirest-java/3.7.04, MIT
     classgraph 4.8.83: https://mvnrepository.com/artifact/io.github.classgraph/classgraph, MIT

@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.e2e.cases.workflow;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.Objects;
 import java.util.UUID;
@@ -46,7 +47,7 @@
 @Slf4j
 public abstract class BaseWorkflowE2ETest {
 
-    protected static String projectName = UUID.randomUUID().toString();
+    protected static final String projectName = UUID.randomUUID().toString();
 
     protected static final AdminUser adminUser = new AdminUser();
 

@@ -42,19 +42,19 @@
 public class NavBarPage {
     protected final RemoteWebDriver driver;
 
-    @FindBy(css = ".tab-horizontal .n-menu-item:nth-child(2) > .n-menu-item-content")
+    @FindBy(xpath = "//div[contains(@class, 'tab-horizontal')]//div[contains(@role,'menubar')]//span[contains(text(), 'Project')]")
     private WebElement projectTab;
 
-    @FindBy(css = ".tab-horizontal .n-menu-item:nth-child(3) > .n-menu-item-content")
+    @FindBy(xpath = "//div[contains(@class, 'tab-horizontal')]//div[contains(@role,'menubar')]//span[contains(text(), 'Resources')]")
     private WebElement resourceTab;
 
-    @FindBy(css = ".tab-horizontal .n-menu-item:nth-child(4) > .n-menu-item-content")
+    @FindBy(xpath = "//div[contains(@class, 'tab-horizontal')]//div[contains(@role,'menubar')]//span[contains(text(), 'Data Quality')]")
     private WebElement dataQualityTab;
 
-    @FindBy(css = ".tab-horizontal .n-menu-item:nth-child(5) > .n-menu-item-content")
+    @FindBy(xpath = "//div[contains(@class, 'tab-horizontal')]//div[contains(@role,'menubar')]//span[contains(text(), 'Datasource')]")
     private WebElement dataSourceTab;
 
-    @FindBy(css = ".tab-horizontal .n-menu-item:nth-child(7) > .n-menu-item-content")
+    @FindBy(xpath = "//div[contains(@class, 'tab-horizontal')]//div[contains(@role,'menubar')]//span[contains(text(), 'Security')]")
     private WebElement securityTab;
 
     public NavBarPage(RemoteWebDriver driver) {
@@ -66,14 +66,14 @@ public NavBarPage(RemoteWebDriver driver) {
     public <T extends NavBarItem> T goToNav(Class<T> nav) {
         if (nav == ProjectPage.class) {
             WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.elementToBeClickable(projectTab));
-            projectTab.click();
+            ((JavascriptExecutor) driver).executeScript("arguments[0].click();", projectTab());
             WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.urlContains("/projects/list"));
             return nav.cast(new ProjectPage(driver));
         }
 
         if (nav == SecurityPage.class) {
             WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.elementToBeClickable(securityTab));
-            securityTab.click();
+            ((JavascriptExecutor) driver).executeScript("arguments[0].click();", securityTab());
             WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.urlContains("/security/tenant-manage"));
             return nav.cast(new SecurityPage(driver));
         }
@@ -87,7 +87,7 @@ public <T extends NavBarItem> T goToNav(Class<T> nav) {
 
         if (nav == DataSourcePage.class) {
             WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.elementToBeClickable(dataSourceTab));
-            dataSourceTab.click();
+            ((JavascriptExecutor) driver).executeScript("arguments[0].click();", dataSourceTab());
             WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.urlContains("/datasource"));
             return nav.cast(new DataSourcePage(driver));
         }

@@ -19,22 +19,19 @@
  */
 package org.apache.dolphinscheduler.e2e.pages.project;
 
-import java.time.Duration;
 import lombok.SneakyThrows;
 import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
 import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
 import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
 import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
 import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab;
 
-import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.remote.RemoteWebDriver;
 import org.openqa.selenium.support.FindBy;
 
 import lombok.Getter;
 import org.openqa.selenium.support.ui.ExpectedConditions;
-import org.openqa.selenium.support.ui.WebDriverWait;
 
 @Getter
 public final class ProjectDetailPage extends NavBarPage {

@@ -110,7 +110,7 @@ public WebElement getTask(String taskName) {
 
     public WorkflowSaveDialog submit() {
         buttonSave().click();
-
+        WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//*[contains(.,'Basic Information')]")));
         return new WorkflowSaveDialog(this);
     }
 

@@ -20,6 +20,8 @@
 package org.apache.dolphinscheduler.e2e.pages.project.workflow;
 
 import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
+
+import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.support.FindBy;
 import org.openqa.selenium.support.PageFactory;
@@ -45,6 +47,8 @@ public WorkflowRunDialog(WorkflowDefinitionTab parent) {
     }
 
     public WorkflowDefinitionTab submit() {
+        By runDialogTitleXpath = By.xpath(String.format("//*[contains(text(), '%s')]", "Please set the parameters before starting"));
+        WebDriverWaitFactory.createWebDriverWait(parent.driver()).until(ExpectedConditions.visibilityOfElementLocated(runDialogTitleXpath));
         WebDriverWaitFactory.createWebDriverWait(parent.driver()).until(ExpectedConditions.elementToBeClickable(buttonSubmit()));
 
         buttonSubmit().click();

@@ -20,7 +20,11 @@
 package org.apache.dolphinscheduler.e2e.pages.project.workflow;
 
 import lombok.Getter;
+
+import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory;
+
 import org.openqa.selenium.By;
+import org.openqa.selenium.JavascriptExecutor;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.support.FindBy;
@@ -44,7 +48,7 @@ public final class WorkflowSaveDialog {
     })
     private WebElement inputName;
 
-    @FindBy(className = "btn-submit")
+    @FindBy(xpath = "//div[contains(text(), 'Basic Information')]/../following-sibling::div[contains(@class, 'n-card__footer')]//button[contains(@class, 'btn-submit')]")
     private WebElement buttonSubmit;
 
     @FindBys({
@@ -72,8 +76,6 @@ public WorkflowSaveDialog name(String name) {
     public WorkflowSaveDialog addGlobalParam(String key, String value) {
         final int len = globalParamsItems().findElements(By.tagName("input")).size();
 
-        final WebDriver driver = parent().driver();
-
         if (len == 0) {
             buttonGlobalCustomParameters().click();
 
@@ -90,8 +92,9 @@ public WorkflowSaveDialog addGlobalParam(String key, String value) {
     }
 
     public WorkflowForm submit() {
-        buttonSubmit().click();
-
+        WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.elementToBeClickable(buttonSubmit));
+        buttonSubmit.click();
+        WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.urlContains("workflow-definition"));
         return parent;
     }
 }
@@ -62,8 +62,6 @@ public class FileManagePage extends NavBarPage implements ResourcePage.Tab {
 
     private final RenameBox renameBox;
 
-    private final CreateFileBox createFileBox;
-
     private final UploadFileBox uploadFileBox;
 
     private final EditFileBox editFileBox;
@@ -90,8 +88,6 @@ public FileManagePage(RemoteWebDriver driver) {
 
         renameBox = new RenameBox();
 
-        createFileBox = new CreateFileBox();
-
         uploadFileBox = new UploadFileBox();
 
         editFileBox = new EditFileBox();
@@ -175,10 +171,11 @@ public FileManagePage createFile(String fileName, String scripts) {
 
         WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.urlContains("/resource/file/create"));
 
-        createFileBox().inputFileName().sendKeys(fileName);
-        createFileBox().codeEditor().content(scripts);
-        createFileBox().buttonSubmit().click();
-        // todo: check if the operation is successful
+        CreateFileBox createFileBox = new CreateFileBox();
+        createFileBox.inputFileName().sendKeys(fileName);
+        createFileBox.codeEditor().content(scripts);
+        createFileBox.buttonSubmit().click();
+        WebDriverWaitFactory.createWebDriverWait(driver).until(ExpectedConditions.urlContains("/resource/file-manage"));
         return this;
     }
 

@@ -79,7 +79,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
     @Override
     @SuppressWarnings("UnstableApiUsage")
     public void beforeAll(ExtensionContext context) throws IOException {
-        Awaitility.setDefaultTimeout(Duration.ofSeconds(60));
+        Awaitility.setDefaultTimeout(Duration.ofSeconds(120));
         Awaitility.setDefaultPollInterval(Duration.ofMillis(500));
 
         setRecordPath();

@@ -211,7 +211,7 @@ worker:
   server-load-protection:
     enabled: true
     # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
-    max-system-cpu-usage-percentage-thresholds: 0.9
+    max-system-cpu-usage-percentage-thresholds: 1
     # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
     max-jvm-cpu-usage-percentage-thresholds: 0.9
     # Worker max System memory usage , when the worker's system memory usage is smaller then this value, worker server can be dispatched tasks.

@@ -26,11 +26,15 @@
 
 import java.io.File;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.google.common.base.Preconditions;
 import com.google.common.io.Files;
 
 public abstract class AbstractStorageOperator implements StorageOperator {
 
+    private static final Logger log = LoggerFactory.getLogger(AbstractStorageOperator.class);
     protected final String resourceBaseAbsolutePath;
 
     public AbstractStorageOperator(String resourceBaseAbsolutePath) {
@@ -60,7 +64,7 @@ public ResourceMetadata getResourceMetaData(String resourceAbsolutePath) {
     @Override
     public String getStorageBaseDirectory() {
         // All directory should end with File.separator
-        return PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
+        return PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/tmp/dolphinscheduler");
     }
 
     @Override