[DSIP-76] Support ssl in netty

.github/workflows/e2e.yml

@@ -200,4 +200,4 @@ jobs:
           if [[ ${{ needs.e2e.result }} != 'success' ]]; then
             echo "E2E Failed!"
             exit -1
-          fi
+          fi

docs/docs/en/guide/installation/cluster.md

@@ -8,6 +8,53 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
 
 Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment.
 
+## Enable SSL (optional)
+
+In cluster deployment, you can enable SSL for secure internal communication. The DolphinScheduler cluster can be configured to use secured communication with internal authentication of the nodes in the cluster.
+To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files.
+
+Step 1: Generate private key (private.pem)
+
+Open the terminal and run the following command to generate a private key:
+
+```bash
+openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048
+```
+
+This command will generate a 2048 bit RSA private key and save it as a private.pem file.
+
+Step 2: Generate Certificate Signing Request (CSR)
+
+Before generating a certificate, you need to generate a Certificate Signing Request (CSR). Run the following command:
+
+```bash
+openssl req -new -key private.pem -out request.csr
+```
+
+This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate.
+
+Step 3: Generate a self signed certificate (cert.crt)
+
+Use CSR to generate self signed certificates. Run the following command:
+
+```bash
+openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt
+```
+
+This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file.
+
+Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, `dolphinscheduler-api` and `dolphinscheduler-alert-server` modules.
+
+```yaml
+rpc:
+  ssl:
+    enabled: true
+    cert-file-path: /path/cert.crt
+    key-file-path: /path/private.pem
+```
+
+You need to change `enabled` to `true` and configure the file routing for `cert-file-path` and `key-file-path`.
+
 ### Prerequisites and DolphinScheduler Startup Environment Preparations
 
 Distribute the installation package to each server of each cluster and perform all the steps in [pseudo-cluster deployment](pseudo-cluster.md) on each machine.

docs/docs/zh/guide/installation/cluster.md

@@ -8,6 +8,54 @@
 
 集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样，所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器，而集群部署(Cluster)需要针对多台机器，且两者“修改相关配置”步骤区别较大
 
+### 开启SSL（可选）
+
+在集群部署中，您可以启用SSL以实现安全的内部通信。DolphinScheduler集群可以配置为使用安全通信，并对集群中的节点进行内部身份验证。
+
+开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。
+
+步骤 1：生成私钥（private.pem）
+
+打开终端并运行以下命令生成私钥：
+
+```bash
+openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048
+```
+
+此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。
+
+步骤 2：生成证书签署请求（CSR）
+
+在生成证书之前，您需要生成一个证书签署请求（CSR）。运行以下命令：
+
+```bash
+openssl req -new -key private.pem -out request.csr
+```
+
+此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。
+
+步骤 3：生成自签名证书（cert.crt）
+
+使用 CSR 来生成自签名证书。运行以下命令：
+
+```bash
+openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt
+```
+
+此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。
+
+然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`、`dolphinscheduler-alert-server`模块中的`application.yaml`文件。
+
+```yaml
+rpc:
+  ssl:
+    enabled: true
+    cert-file-path: /path/cert.crt
+    key-file-path: /path/private.pem
+```
+
+您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。
+
 ### 前置准备工作 && 准备 DolphinScheduler 启动环境
 
 需要将安装包分发至每台集群的每台服务器上，并且需要在每台机器中进行配置执行[伪集群部署](pseudo-cluster.md)中的所有执行项

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java

@@ -25,6 +25,7 @@
 import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.dao.DaoConfiguration;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
 import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
 
 import javax.annotation.PostConstruct;
@@ -40,7 +41,8 @@
 @Slf4j
 @Import({CommonConfiguration.class,
         DaoConfiguration.class,
-        RegistryConfiguration.class})
+        RegistryConfiguration.class,
+        NettySslConfig.class})
 @SpringBootApplication
 public class AlertServer {
 

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java

@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.alert.config;
 
 import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -47,6 +48,8 @@ public final class AlertConfig implements Validator {
 
     private String alertServerAddress;
 
+    private NettySslConfig nettySslConfig;
+
     @Override
     public boolean supports(Class<?> clazz) {
         return AlertConfig.class.isAssignableFrom(clazz);

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml

@@ -75,6 +75,11 @@ alert:
   max-heartbeat-interval: 60s
   # The maximum number of alerts that can be processed in parallel
   sender-parallelism: 100
+  rpc:
+    ssl:
+      enabled: false
+      cert-file-path: /path/cert.crt
+      key-file-path: /path/private.pem
 
 registry:
   type: zookeeper

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java

@@ -23,6 +23,7 @@
 import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
 import org.apache.dolphinscheduler.dao.DaoConfiguration;
 import org.apache.dolphinscheduler.dao.PluginDao;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
 import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
 import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
@@ -44,14 +45,18 @@
         CommonConfiguration.class,
         ServiceConfiguration.class,
         StorageConfiguration.class,
-        RegistryConfiguration.class})
+        RegistryConfiguration.class,
+        NettySslConfig.class})
 @ServletComponentScan
 @SpringBootApplication
 public class ApiApplicationServer {
 
     @Autowired
     private PluginDao pluginDao;
 
+    @Autowired
+    NettySslConfig nettySslConfig;
+
     public static void main(String[] args) {
         ApiServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount);
         Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance());

dolphinscheduler-api/src/main/resources/application.yaml

@@ -164,6 +164,11 @@ api:
     # Close each active connection of socket server if python program not active after x milliseconds. Define value is
     # (0 = infinite), and socket server would never close even though no requests accept
     read-timeout: 0
+  rpc:
+    ssl:
+      enabled: false
+      cert-file-path: /path/cert.crt
+      key-file-path: /path/private.pem
 
 metrics:
   enabled: true

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.e2e.cases.cluster;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest;
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.core.WebDriverHolder;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm;
+import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
+import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
+import org.apache.dolphinscheduler.e2e.pages.security.UserPage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junitpioneer.jupiter.DisableIfTestFails;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yaml")
+@DisableIfTestFails
+public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest {
+
+    @BeforeAll
+    public static void setup() {
+        browser = WebDriverHolder.getWebDriver();
+
+        TenantPage tenantPage = new LoginPage(browser)
+                .login(adminUser)
+                .goToNav(SecurityPage.class)
+                .goToTab(TenantPage.class);
+
+        if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) {
+            tenantPage
+                    .create(adminUser.getTenant())
+                    .goToNav(SecurityPage.class)
+                    .goToTab(UserPage.class)
+                    .update(adminUser);
+        }
+
+        tenantPage
+                .goToNav(ProjectPage.class)
+                .createProjectUntilSuccess(projectName);
+    }
+
+    @Test
+    void testRunShellTasks_SuccessCase() {
+        WorkflowDefinitionTab workflowDefinitionPage =
+                new ProjectPage(browser)
+                        .goToNav(ProjectPage.class)
+                        .goTo(projectName)
+                        .goToTab(WorkflowDefinitionTab.class);
+
+        // todo: use yaml to define the workflow
+        String workflowName = "SslSuccessCase";
+        String taskName = "SslShellSuccess";
+        workflowDefinitionPage
+                .createWorkflow()
+                .<ShellTaskForm>addTask(WorkflowForm.TaskType.SHELL)
+                .script("echo hello world\n")
+                .name(taskName)
+                .submit()
+
+                .submit()
+                .name(workflowName)
+                .submit();
+
+        untilWorkflowDefinitionExist(workflowName);
+
+        workflowDefinitionPage.publish(workflowName);
+
+        runWorkflow(workflowName);
+        untilWorkflowInstanceExist(workflowName);
+        WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
+        assertThat(workflowInstance.executionTime()).isEqualTo(1);
+
+        TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
+        assertThat(taskInstance.retryTimes()).isEqualTo(0);
+    }
+
+}

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM eclipse-temurin:8-jdk
+
+ENV DOCKER=true
+ENV TZ=Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler
+
+RUN apt update ; \
+    apt install -y sudo ; \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+COPY /tmp/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+    rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+    mv apache-dolphinscheduler-*-bin/* . ; \
+    rm -rf apache-dolphinscheduler-*-bin