diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index eb43d4059258..22b7783e280c 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -200,4 +200,4 @@ jobs:
if [[ ${{ needs.e2e.result }} != 'success' ]]; then
echo "E2E Failed!"
exit -1
- fi
+ fi
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index f30edcd8a1a3..ee2912d17902 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -13,4 +13,4 @@
-
+
\ No newline at end of file
diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md
index 5dfc65699f2b..301123153a71 100644
--- a/docs/docs/en/guide/installation/cluster.md
+++ b/docs/docs/en/guide/installation/cluster.md
@@ -8,6 +8,53 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment.
+## Enable SSL (optional)
+
+In cluster deployment, you can enable SSL for secure internal communication. The DolphinScheduler cluster can be configured to use secured communication with internal authentication of the nodes in the cluster.
+To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files.
+
+Step 1: Generate private key (private.pem)
+
+Open the terminal and run the following command to generate a private key:
+
+```bash
+openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048
+```
+
+This command will generate a 2048 bit RSA private key and save it as a private.pem file.
+
+Step 2: Generate Certificate Signing Request (CSR)
+
+Before generating a certificate, you need to generate a Certificate Signing Request (CSR). Run the following command:
+
+```bash
+openssl req -new -key private.pem -out request.csr
+```
+
+This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate.
+
+Step 3: Generate a self signed certificate (cert.crt)
+
+Use CSR to generate self signed certificates. Run the following command:
+
+```bash
+openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt
+```
+
+This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file.
+
+Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, `dolphinscheduler-api` and `dolphinscheduler-alert-server` modules.
+
+```yaml
+rpc:
+ ssl:
+ enabled: true
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
+```
+
+You need to change `enabled` to `true` and configure the file routing for `cert-file-path` and `key-file-path`.
+
### Prerequisites and DolphinScheduler Startup Environment Preparations
Distribute the installation package to each server of each cluster and perform all the steps in [pseudo-cluster deployment](pseudo-cluster.md) on each machine.
diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md
index 5ed407ca1860..c5aed2e3f204 100644
--- a/docs/docs/zh/guide/installation/cluster.md
+++ b/docs/docs/zh/guide/installation/cluster.md
@@ -8,6 +8,54 @@
集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样,所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器,而集群部署(Cluster)需要针对多台机器,且两者“修改相关配置”步骤区别较大
+### 开启SSL(可选)
+
+在集群部署中,您可以启用SSL以实现安全的内部通信。DolphinScheduler集群可以配置为使用安全通信,并对集群中的节点进行内部身份验证。
+
+开启SLL认证,你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。
+
+步骤 1:生成私钥(private.pem)
+
+打开终端并运行以下命令生成私钥:
+
+```bash
+openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048
+```
+
+此命令会生成一个 2048 位的 RSA 私钥,并将其保存为 private.pem 文件。
+
+步骤 2:生成证书签署请求(CSR)
+
+在生成证书之前,您需要生成一个证书签署请求(CSR)。运行以下命令:
+
+```bash
+openssl req -new -key private.pem -out request.csr
+```
+
+此命令会提示您输入一些信息,例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。
+
+步骤 3:生成自签名证书(cert.crt)
+
+使用 CSR 来生成自签名证书。运行以下命令:
+
+```bash
+openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt
+```
+
+此命令会生成一个有效期为 365 天的自签名证书,并将其保存为 cert.crt 文件。
+
+然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`、`dolphinscheduler-alert-server`模块中的`application.yaml`文件。
+
+```yaml
+rpc:
+ ssl:
+ enabled: true
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
+```
+
+您需要将`enabled`改为`true`,同时将配置`cert-file-path`和`key-file-path`的文件路劲。
+
### 前置准备工作 && 准备 DolphinScheduler 启动环境
需要将安装包分发至每台集群的每台服务器上,并且需要在每台机器中进行配置执行[伪集群部署](pseudo-cluster.md)中的所有执行项
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
index ff2e98542629..6e3688e6cc75 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
@@ -25,6 +25,7 @@
import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.dao.DaoConfiguration;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
import javax.annotation.PostConstruct;
@@ -40,7 +41,8 @@
@Slf4j
@Import({CommonConfiguration.class,
DaoConfiguration.class,
- RegistryConfiguration.class})
+ RegistryConfiguration.class,
+ NettySslConfig.class})
@SpringBootApplication
public class AlertServer {
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java
index 240f92b846f8..a5217ed9d510 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.alert.config;
import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.commons.lang3.StringUtils;
@@ -47,6 +48,8 @@ public final class AlertConfig implements Validator {
private String alertServerAddress;
+ private NettySslConfig nettySslConfig;
+
@Override
public boolean supports(Class> clazz) {
return AlertConfig.class.isAssignableFrom(clazz);
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
index 927cbc3c2ce6..7251d9becb18 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
@@ -75,6 +75,11 @@ alert:
max-heartbeat-interval: 60s
# The maximum number of alerts that can be processed in parallel
sender-parallelism: 100
+ rpc:
+ ssl:
+ enabled: false
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
registry:
type: zookeeper
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
index b072fe630878..8341a1cbb59f 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
@@ -23,6 +23,7 @@
import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
import org.apache.dolphinscheduler.dao.DaoConfiguration;
import org.apache.dolphinscheduler.dao.PluginDao;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider;
import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
@@ -44,7 +45,8 @@
CommonConfiguration.class,
ServiceConfiguration.class,
StorageConfiguration.class,
- RegistryConfiguration.class})
+ RegistryConfiguration.class,
+ NettySslConfig.class})
@ServletComponentScan
@SpringBootApplication
public class ApiApplicationServer {
@@ -52,6 +54,9 @@ public class ApiApplicationServer {
@Autowired
private PluginDao pluginDao;
+ @Autowired
+ NettySslConfig nettySslConfig;
+
public static void main(String[] args) {
ApiServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount);
Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance());
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
index 9b0e94d64451..8ce78518c571 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -164,6 +164,11 @@ api:
# Close each active connection of socket server if python program not active after x milliseconds. Define value is
# (0 = infinite), and socket server would never close even though no requests accept
read-timeout: 0
+ rpc:
+ ssl:
+ enabled: false
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
metrics:
enabled: true
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java
new file mode 100644
index 000000000000..728b01f4eb12
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.e2e.cases.cluster;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest;
+import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
+import org.apache.dolphinscheduler.e2e.core.WebDriverHolder;
+import org.apache.dolphinscheduler.e2e.pages.LoginPage;
+import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab;
+import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm;
+import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
+import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
+import org.apache.dolphinscheduler.e2e.pages.security.UserPage;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junitpioneer.jupiter.DisableIfTestFails;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yaml")
+@DisableIfTestFails
+public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest {
+
+ @BeforeAll
+ public static void setup() {
+ browser = WebDriverHolder.getWebDriver();
+
+ TenantPage tenantPage = new LoginPage(browser)
+ .login(adminUser)
+ .goToNav(SecurityPage.class)
+ .goToTab(TenantPage.class);
+
+ if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) {
+ tenantPage
+ .create(adminUser.getTenant())
+ .goToNav(SecurityPage.class)
+ .goToTab(UserPage.class)
+ .update(adminUser);
+ }
+
+ tenantPage
+ .goToNav(ProjectPage.class)
+ .createProjectUntilSuccess(projectName);
+ }
+
+ @Test
+ void testRunShellTasks_SuccessCase() {
+ WorkflowDefinitionTab workflowDefinitionPage =
+ new ProjectPage(browser)
+ .goToNav(ProjectPage.class)
+ .goTo(projectName)
+ .goToTab(WorkflowDefinitionTab.class);
+
+ // todo: use yaml to define the workflow
+ String workflowName = "SslSuccessCase";
+ String taskName = "SslShellSuccess";
+ workflowDefinitionPage
+ .createWorkflow()
+ .addTask(WorkflowForm.TaskType.SHELL)
+ .script("echo hello world\n")
+ .name(taskName)
+ .submit()
+
+ .submit()
+ .name(workflowName)
+ .submit();
+
+ untilWorkflowDefinitionExist(workflowName);
+
+ workflowDefinitionPage.publish(workflowName);
+
+ runWorkflow(workflowName);
+ untilWorkflowInstanceExist(workflowName);
+ WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName);
+ assertThat(workflowInstance.executionTime()).isEqualTo(1);
+
+ TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName);
+ assertThat(taskInstance.retryTimes()).isEqualTo(0);
+ }
+
+}
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile
new file mode 100644
index 000000000000..55aa5392cc0c
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM eclipse-temurin:8-jdk
+
+ENV DOCKER=true
+ENV TZ=Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y sudo ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+COPY /tmp/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME
+RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \
+ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \
+ mv apache-dolphinscheduler-*-bin/* . ; \
+ rm -rf apache-dolphinscheduler-*-bin
\ No newline at end of file
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml
new file mode 100644
index 000000000000..8305faa6fd3c
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml
@@ -0,0 +1,254 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 12345
+ servlet:
+ session:
+ timeout: 120m
+ context-path: /dolphinscheduler/
+ compression:
+ enabled: true
+ mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
+ jetty:
+ max-http-form-post-size: 5000000
+ accesslog:
+ enabled: true
+ custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms'
+
+spring:
+ profiles:
+ active: postgresql
+ banner:
+ charset: UTF-8
+ jackson:
+ time-zone: UTC
+ date-format: "yyyy-MM-dd HH:mm:ss"
+ servlet:
+ multipart:
+ max-file-size: 1024MB
+ max-request-size: 1024MB
+ messages:
+ basename: i18n/messages
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ pool-name: DolphinScheduler
+ quartz:
+ auto-startup: false
+ job-store-type: jdbc
+ jdbc:
+ initialize-schema: never
+ properties:
+ org.quartz.jobStore.isClustered: true
+ org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
+ org.quartz.scheduler.instanceId: AUTO
+ org.quartz.jobStore.tablePrefix: QRTZ_
+ org.quartz.jobStore.acquireTriggersWithinLock: true
+ org.quartz.scheduler.instanceName: DolphinScheduler
+ org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool
+ org.quartz.jobStore.useProperties: false
+ org.quartz.jobStore.misfireThreshold: 60000
+ org.quartz.scheduler.makeSchedulerThreadDaemon: true
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+ org.quartz.jobStore.clusterCheckinInterval: 5000
+ org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
+ mvc:
+ pathmatch:
+ matching-strategy: ANT_PATH_MATCHER
+ static-path-pattern: /static/**
+ cloud.discovery.client.composite-indicator.enabled: false
+
+springdoc:
+ swagger-ui:
+ path: /swagger-ui.html
+ packages-to-scan: org.apache.dolphinscheduler.api
+
+# Mybatis-plus configuration, you don't need to change it
+mybatis-plus:
+ mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
+ type-aliases-package: org.apache.dolphinscheduler.dao.entity
+ configuration:
+ cache-enabled: false
+ call-setters-on-nulls: true
+ map-underscore-to-camel-case: true
+ jdbc-type-for-null: NULL
+ global-config:
+ db-config:
+ id-type: auto
+ banner: false
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: health,metrics,prometheus
+ endpoint:
+ health:
+ enabled: true
+ show-details: always
+ health:
+ db:
+ enabled: true
+ defaults:
+ enabled: false
+ metrics:
+ tags:
+ application: ${spring.application.name}
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 1s
+ max-sleep: 3s
+ max-retries: 5
+ session-timeout: 60s
+ connection-timeout: 15s
+ block-until-connected: 15s
+ digest: ~
+
+api:
+ audit-enable: false
+ # Traffic control, if you turn on this config, the maximum number of request/s will be limited.
+ # global max request number per second
+ # default tenant-level max request number
+ traffic-control:
+ global-switch: false
+ max-global-qps-rate: 300
+ tenant-switch: false
+ default-tenant-qps-rate: 10
+ #customize-tenant-qps-rate:
+ # eg.
+ #tenant1: 11
+ #tenant2: 20
+ python-gateway:
+ # Weather enable python gateway server or not. The default value is false.
+ enabled: false
+ # Authentication token for connection from python api to python gateway server. Should be changed the default value
+ # when you deploy in public network.
+ auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
+ # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
+ # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
+ gateway-server-address: 0.0.0.0
+ # The port of Python gateway server start. Define which port you could connect to Python gateway server from
+ # Python API side.
+ gateway-server-port: 25333
+ # The address of Python callback client.
+ python-address: 127.0.0.1
+ # The port of Python callback client.
+ python-port: 25334
+ # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
+ # and socket server would never close even though no requests accept
+ connect-timeout: 0
+ # Close each active connection of socket server if python program not active after x milliseconds. Define value is
+ # (0 = infinite), and socket server would never close even though no requests accept
+ read-timeout: 0
+ rpc:
+ ssl:
+ enabled: true
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
+
+metrics:
+ enabled: true
+
+security:
+ authentication:
+ # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
+ type: PASSWORD
+ # IF you set type `LDAP`, below config will be effective
+ ldap:
+ # ldap server config
+ urls: ldap://ldap.forumsys.com:389/
+ base-dn: dc=example,dc=com
+ username: cn=read-only-admin,dc=example,dc=com
+ password: password
+ user:
+ # admin userId when you use LDAP login
+ admin: read-only-admin
+ identity-attribute: uid
+ email-attribute: mail
+ # action when ldap user is not exist (supported types: CREATE,DENY)
+ not-exist-action: CREATE
+ ssl:
+ enable: false
+ # jks file absolute path && password
+ trust-store: "/ldapkeystore.jks"
+ trust-store-password: "password"
+ casdoor:
+ user:
+ admin: ""
+ oauth2:
+ enable: false
+ provider:
+ github:
+ authorizationUri: ""
+ redirectUri: ""
+ clientId: ""
+ clientSecret: ""
+ tokenUri: ""
+ userInfoUri: ""
+ callbackUrl: ""
+ iconUri: ""
+ provider: github
+ google:
+ authorizationUri: ""
+ redirectUri: ""
+ clientId: ""
+ clientSecret: ""
+ tokenUri: ""
+ userInfoUri: ""
+ callbackUrl: ""
+ iconUri: ""
+ provider: google
+casdoor:
+ # Your Casdoor server url
+ endpoint: ""
+ client-id: ""
+ client-secret: ""
+ # The certificate may be multi-line, you can use `|-` for ease
+ certificate: ""
+ # Your organization name added in Casdoor
+ organization-name: ""
+ # Your application name added in Casdoor
+ application-name: ""
+ # Doplhinscheduler login url
+ redirect-url: ""
+
+
+# Override by profile
+
+---
+spring:
+ config:
+ activate:
+ on-profile: mysql
+ datasource:
+ driver-class-name: com.mysql.cj.jdbc.Driver
+ url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+ username: root
+ password: root
+ quartz:
+ properties:
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml
new file mode 100644
index 000000000000..1739eabd10ac
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml
@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.8"
+
+services:
+ dolphinscheduler-zookeeper:
+ image: bitnami/zookeeper:3.7.1
+ environment:
+ ALLOW_ANONYMOUS_LOGIN: "yes"
+ ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
+ volumes:
+ - dolphinscheduler-zookeeper:/bitnami/zookeeper
+ healthcheck:
+ test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - dolphinscheduler
+ command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"]
+
+ dolphinscheduler-api:
+ image: apache/dolphinscheduler-api:ci
+ environment:
+ MASTER_MAX_CPU_LOAD_AVG: 100
+ WORKER_TENANT_AUTO_CREATE: 'true'
+ ports:
+ - "12345:12345"
+ - "25333:25333"
+ networks:
+ - dolphinscheduler
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ]
+ interval: 5s
+ timeout: 5s
+ retries: 10
+ depends_on:
+ dolphinscheduler-zookeeper:
+ condition: service_completed_successfully
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ]
+
+ dolphinscheduler-alert:
+ image: apache/dolphinscheduler-alert-server:ci
+ environment:
+ MASTER_MAX_CPU_LOAD_AVG: 100
+ WORKER_TENANT_AUTO_CREATE: 'true'
+ networks:
+ - dolphinscheduler
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
+ interval: 5s
+ timeout: 5s
+ retries: 10
+ depends_on:
+ dolphinscheduler-zookeeper:
+ condition: service_completed_successfully
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ]
+
+ dolphinscheduler-master:
+ image: apache/dolphinscheduler-master:ci
+ environment:
+ MASTER_MAX_CPU_LOAD_AVG: 100
+ WORKER_TENANT_AUTO_CREATE: 'true'
+ networks:
+ - dolphinscheduler
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ]
+ interval: 5s
+ timeout: 5s
+ retries: 10
+ depends_on:
+ dolphinscheduler-zookeeper:
+ condition: service_completed_successfully
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ]
+
+ dolphinscheduler-worker:
+ image: apache/dolphinscheduler-worker:ci
+ environment:
+ MASTER_MAX_CPU_LOAD_AVG: 100
+ WORKER_TENANT_AUTO_CREATE: 'true'
+ networks:
+ - dolphinscheduler
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ]
+ interval: 5s
+ timeout: 5s
+ retries: 10
+ depends_on:
+ dolphinscheduler-zookeeper:
+ condition: service_completed_successfully
+ volumes:
+ - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ]
+
+networks:
+ dolphinscheduler:
+
+volumes:
+ dolphinscheduler-postgresql:
+ dolphinscheduler-zookeeper:
+ dolphinscheduler-worker-data:
+ dolphinscheduler-logs:
+ dolphinscheduler-shared-local:
+ dolphinscheduler-resource-local:
\ No newline at end of file
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml
new file mode 100644
index 000000000000..979d54a02221
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml
@@ -0,0 +1,168 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+spring:
+ profiles:
+ active: postgresql
+ banner:
+ charset: UTF-8
+ jackson:
+ time-zone: UTC
+ date-format: "yyyy-MM-dd HH:mm:ss"
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ pool-name: DolphinScheduler
+ quartz:
+ job-store-type: jdbc
+ jdbc:
+ initialize-schema: never
+ properties:
+ org.quartz.threadPool.threadPriority: 5
+ org.quartz.jobStore.isClustered: true
+ org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
+ org.quartz.scheduler.instanceId: AUTO
+ org.quartz.jobStore.tablePrefix: QRTZ_
+ org.quartz.jobStore.acquireTriggersWithinLock: true
+ org.quartz.scheduler.instanceName: DolphinScheduler
+ org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
+ org.quartz.jobStore.useProperties: false
+ org.quartz.threadPool.makeThreadsDaemons: true
+ org.quartz.threadPool.threadCount: 25
+ org.quartz.jobStore.misfireThreshold: 60000
+ org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
+ org.quartz.scheduler.makeSchedulerThreadDaemon: true
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+ org.quartz.jobStore.clusterCheckinInterval: 5000
+ cloud.discovery.client.composite-indicator.enabled: false
+
+# Mybatis-plus configuration, you don't need to change it
+mybatis-plus:
+ mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
+ type-aliases-package: org.apache.dolphinscheduler.dao.entity
+ configuration:
+ cache-enabled: false
+ call-setters-on-nulls: true
+ map-underscore-to-camel-case: true
+ jdbc-type-for-null: NULL
+ global-config:
+ db-config:
+ id-type: auto
+ banner: false
+
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 1s
+ max-sleep: 3s
+ max-retries: 5
+ session-timeout: 60s
+ connection-timeout: 15s
+ block-until-connected: 15s
+ digest: ~
+
+master:
+ listen-port: 5678
+ # The number of workflow event bus fire worker used to fire the workflow event, default is 2*CPU core + 1.
+ # workflow-event-bus-fire-thread-count: 10
+ # The number of threads used to execute sync logic task e.g. Switch/Condition, default is CPU core.
+ # master-sync-task-executor-thread-pool-size: 10
+ # The number of threads used to execute async logic task e.g. Dependent/SubWorkflow default is CPU core.
+ # master-async-task-executor-thread-pool-size: 10
+ max-heartbeat-interval: 10s
+ server-load-protection:
+ # If set true, will open master overload protection
+ enabled: true
+ # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
+ max-system-cpu-usage-percentage-thresholds: 0.7
+ # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
+ max-jvm-cpu-usage-percentage-thresholds: 0.7
+ # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
+ max-system-memory-usage-percentage-thresholds: 0.7
+ # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
+ max-disk-usage-percentage-thresholds: 0.7
+ registry-disconnect-strategy:
+ # The disconnect strategy: stop, waiting
+ strategy: stop
+ worker-group-refresh-interval: 10s
+ command-fetch-strategy:
+ type: ID_SLOT_BASED
+ config:
+ # The incremental id step
+ id-step: 1
+ # master fetch command num
+ fetch-size: 10
+ worker-load-balancer-configuration-properties:
+ # RANDOM, ROUND_ROBIN, FIXED_WEIGHTED_ROUND_ROBIN, DYNAMIC_WEIGHTED_ROUND_ROBIN
+ type: DYNAMIC_WEIGHTED_ROUND_ROBIN
+ # dynamic-weight-config-properties only used in DYNAMIC_WEIGHTED_ROUND_ROBIN, the weight of memory-usage, cpu-usage, task-thread-pool-usage should sum to 100.
+ dynamic-weight-config-properties:
+ memory-usage-weight: 30
+ cpu-usage-weight: 30
+ task-thread-pool-usage-weight: 40
+ rpc:
+ ssl:
+ enabled: true
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
+
+server:
+ port: 5679
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: health,metrics,prometheus
+ endpoint:
+ health:
+ enabled: true
+ show-details: always
+ health:
+ db:
+ enabled: true
+ defaults:
+ enabled: false
+ metrics:
+ tags:
+ application: ${spring.application.name}
+
+metrics:
+ enabled: true
+
+# Override by profile
+
+---
+spring:
+ config:
+ activate:
+ on-profile: mysql
+ datasource:
+ driver-class-name: com.mysql.cj.jdbc.Driver
+ url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+ username: root
+ password: root
+ quartz:
+ properties:
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt
new file mode 100644
index 000000000000..12910f535382
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-----BEGIN CERTIFICATE-----
+MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC
+MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu
+ZzEgMB4GA1UECgwXYXBhY2hlIGRvbHBoaW5zY2hlZHVsZXIxFjAUBgNVBAsMDUlU
+IERlcGFydG1lbnQxGTAXBgNVBAMMEGRvbHBoaW5zY2hlZHVsZXIxOjA4BgkqhkiG
+9w0BCQEWK3VzZXJzLXN1YnNjcmliZUBkb2xwaGluc2NoZWR1bGVyLmFwYWNoZS5v
+cmcwHhcNMjQxMDE2MDcyODQ0WhcNMjUxMDE2MDcyODQ0WjCBwjELMAkGA1UEBhMC
+Q04xEDAOBgNVBAgMB2JlaWppbmcxEDAOBgNVBAcMB2JlaWppbmcxIDAeBgNVBAoM
+F2FwYWNoZSBkb2xwaGluc2NoZWR1bGVyMRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50
+MRkwFwYDVQQDDBBkb2xwaGluc2NoZWR1bGVyMTowOAYJKoZIhvcNAQkBFit1c2Vy
+cy1zdWJzY3JpYmVAZG9scGhpbnNjaGVkdWxlci5hcGFjaGUub3JnMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwr5KUI0ZzVcpOjazaZUDo+y+uEnt1qRC
+cmnnNFyMQuVhX5gBHRk+As5e+E+M3TTLpA4LFti644NqN9eUmuqTzEUAfRzd2dfh
+cEKFU3TlhpmsLQ7nN/gYe0KMmh9imV7WFV25NPUsiGgEKB5EEFQkq0Ii6A7GHDUD
+Cmwcchkwbgr4wCUVJfAjzfSsq1WuNnGlCeIzv9Xv/drqQ2StWuoDav2YoPsMqcTU
+yXnrWWUW/L7TLMHZSe7uNphul3BRXNx6c7J9k9QRTDuyJt0seRM7PgFK/9aVKXxo
+BG+k2pGLw+Xoq4URNS09YXsBiqEDy5A+RBUivfyE6nGFblraa6I++wIDAQABMA0G
+CSqGSIb3DQEBCwUAA4IBAQA7LHY1apCsncI+GcDqvfClWBqJLyCCzLnQ4Z+vLsvC
+C0zB0MXpqjXXe/FXcU62DNIXr0IyOlZTApuM7sLZftVYAihGzdMxQuhhoa1yZPo4
+2O6RulbRpxkFBNyfvOAGdmRi+pEfdXX2o/lqzgMqR2vzG79aOUYbgwPXIis4hZOP
+Zp1Asz2JojmIRXXYVz00upCIuPs31D/0wZim6RG9RNqToiFCFc+wKENN/fhVwaSC
+mz3GKWnagQA7JR6Dz9dVqqPtkhOaSW6KajFU7rvhg/C0VH8i2devOgAW6/7X6e2x
++XHg8jQ6GaYqDdqFk4T9Zpa/aMHBrnB0aHrrTJVrPSei
+-----END CERTIFICATE-----
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem
new file mode 100644
index 000000000000..ba90f817b5c7
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6
+NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3
+15Sa6pPMRQB9HN3Z1+FwQoVTdOWGmawtDuc3+Bh7QoyaH2KZXtYVXbk09SyIaAQo
+HkQQVCSrQiLoDsYcNQMKbBxyGTBuCvjAJRUl8CPN9KyrVa42caUJ4jO/1e/92upD
+ZK1a6gNq/Zig+wypxNTJeetZZRb8vtMswdlJ7u42mG6XcFFc3Hpzsn2T1BFMO7Im
+3Sx5Ezs+AUr/1pUpfGgEb6TakYvD5eirhRE1LT1hewGKoQPLkD5EFSK9/ITqcYVu
+Wtproj77AgMBAAECggEAV2e+BgpWftW0gNbFTsmdgSkEbsVrtM8zrZN/vH4fS/UU
+76lTMqK+NNc4Q928fRoxnjfZXU84Dl5QwbCGF8PaQy/jKMRYhGpC2jZZDdylGfUG
+FJEgUPwOzN/RKGmWwPySPLO8qRqzeqOo3fK7nNKPRLhh32KmI0ndzAM6/zD0Dich
+CzGO+liQbhKNoBUXlZXhJs9Bu9mvkL4yId1eks8RZAt/Opi2CoijhLhs5ncdGRR9
+oB+D8KSSyuNo3+QQBoKLz++snC6EI/NTSRM86wDroVnYVxL7mXzmQ2fWNoUmAsX4
+ugXUAmCCz1d66tAJst0l30qMykiIT/jQg4uT22ecpQKBgQDiwfBMK+t0kzcZKiNF
+u7jFWNGIHnJ4EU8xdzSbaWujuE7zP0NvC0e4Ej0DiJ7S4OBhbBuSDTSl0BQUfpXZ
+O/UlSNJvgvCCjGXHnvtsXzza5fqvxjhGIbO/XOhSZZhriG1YDbCtitvfP3xR85EV
+Os7/2KfztR2bFH66+lj51zYn3QKBgQDb23L443avaBxD4v9DKtGIP8JS7TPm4s39
+MgluzOvupH4dWZs83RSbsBa3AkzDvU268gRjwZYLFRwld13B8sfMWEMxTVX3iliZ
+58GH/4DB7ixoMLU2yJglIQ8gKrJs3nJyUh2ilU7j2HWGlRvXlMgLFrgR0udgXmnP
+un9+4L7AtwKBgQCeGBDtSoGymSEFDnfhGmxMSwxrFq9APvEzz6SB5hIB53JD7Yua
+VAcAstaPLCVvaRGHWtQuE77tDBGz6D2ZY1AbWpY84NHGcxRTBOjOOeg3xbft7Qk2
+/DBR7via2fJWppt1lLXliLabA6wtht9UKdhAUFFofJHTmYpx2jFGn2FGkQKBgAcc
+qjNy6Dv0WjW3+hnqIK0PovGbyroFTDSMkdGLv3YQzCOW6nalkC0ZHSL+aA1M6l7J
+xWqUsI3bOPXdTqMff2cU09eccNN5HjScUp5bnYaodbdJHBORmZpZ32ti9jKfX4Rj
+M6I2wj6MmgCogiYPIgl5yqqI8AMmCrY1ijpIeuMxAoGAdeisS8UakUdn0lVE5NSp
+stncEwnjggL5gzVAHSQ/vzSddk7mUxCWZu//GryEJeFS7Q4jhBE7z5Y6GceiBraC
+6YCauAAExswgi2wpxW9dwOOY1+zKVmUIoklUJ9nIuYFSxY/pDf1rQWUOUa3x104W
+FYIRNxNXs7dGOs7Ge4M01Uw=
+-----END PRIVATE KEY-----
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml
new file mode 100644
index 000000000000..8965b044dcbf
--- /dev/null
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+spring:
+ banner:
+ charset: UTF-8
+ jackson:
+ time-zone: UTC
+ date-format: "yyyy-MM-dd HH:mm:ss"
+ autoconfigure:
+ exclude:
+ - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
+ cloud.discovery.client.composite-indicator.enabled: false
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 1s
+ max-sleep: 3s
+ max-retries: 5
+ session-timeout: 60s
+ connection-timeout: 15s
+ block-until-connected: 15s
+ digest: ~
+
+worker:
+ # worker listener port
+ listen-port: 1234
+ # worker execute thread number to limit task instances in parallel
+ exec-threads: 100
+ # worker heartbeat interval
+ max-heartbeat-interval: 10s
+ # worker host weight to dispatch tasks, default value 100
+ host-weight: 100
+ server-load-protection:
+ # If set true, will open worker overload protection
+ enabled: true
+ # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
+ max-system-cpu-usage-percentage-thresholds: 0.7
+ # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
+ max-jvm-cpu-usage-percentage-thresholds: 0.7
+ # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
+ max-system-memory-usage-percentage-thresholds: 0.7
+ # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
+ max-disk-usage-percentage-thresholds: 0.7
+ registry-disconnect-strategy:
+ # The disconnect strategy: stop, waiting
+ strategy: stop
+ task-execute-threads-full-policy: REJECT
+ tenant-config:
+ # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
+ auto-create-tenant-enabled: true
+ # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.
+ default-tenant-enabled: false
+ rpc:
+ ssl:
+ enabled: true
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
+
+server:
+ port: 1235
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: health,metrics,prometheus
+ endpoint:
+ health:
+ enabled: true
+ show-details: always
+ health:
+ db:
+ enabled: true
+ defaults:
+ enabled: false
+ metrics:
+ tags:
+ application: ${spring.application.name}
+
+metrics:
+ enabled: true
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java
index 4aea4d6dfe9c..fed57160d239 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java
@@ -21,6 +21,7 @@
import org.apache.dolphinscheduler.extract.base.IRpcResponse;
import org.apache.dolphinscheduler.extract.base.SyncRequestDto;
import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.dolphinscheduler.extract.base.exception.RemotingException;
import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException;
import org.apache.dolphinscheduler.extract.base.future.ResponseFuture;
@@ -33,6 +34,7 @@
import org.apache.dolphinscheduler.extract.base.utils.Host;
import org.apache.dolphinscheduler.extract.base.utils.NettyUtils;
+import java.io.File;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -41,6 +43,8 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
+import javax.net.ssl.SSLException;
+
import lombok.extern.slf4j.Slf4j;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
@@ -52,6 +56,8 @@
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.timeout.IdleStateHandler;
@Slf4j
@@ -70,8 +76,20 @@ public class NettyRemotingClient implements AutoCloseable {
private final NettyClientHandler clientHandler;
+ private SslContext sslContext = null;
+
public NettyRemotingClient(final NettyClientConfig clientConfig) {
this.clientConfig = clientConfig;
+ NettySslConfig nettySslConfig = clientConfig.getNettySslConfig();
+ if (nettySslConfig.isEnabled()) {
+ try {
+ sslContext =
+ SslContextBuilder.forClient().trustManager(new File(nettySslConfig.getCertFilePath())).build();
+ } catch (SSLException e) {
+ throw new IllegalArgumentException("Initialize SslContext error, please check the cert-file", e);
+ }
+ }
+
ThreadFactory nettyClientThreadFactory = ThreadUtils.newDaemonThreadFactory("NettyClientThread-");
if (Epoll.isAvailable()) {
this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), nettyClientThreadFactory);
@@ -97,6 +115,10 @@ private void start() {
@Override
public void initChannel(SocketChannel ch) {
+ NettySslConfig nettySslConfig = clientConfig.getNettySslConfig();
+ if (nettySslConfig.isEnabled()) {
+ ch.pipeline().addLast(sslContext.newHandler(ch.alloc()));
+ }
ch.pipeline()
.addLast("client-idle-handler",
new IdleStateHandler(
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
index a00ff540f4a5..974f9c43468c 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
@@ -76,4 +76,7 @@ public class NettyClientConfig {
@Builder.Default
private int defaultRpcTimeoutMillis = 10_000;
+ @Builder.Default
+ private NettySslConfig nettySslConfig = new NettySslConfig();
+
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java
new file mode 100644
index 000000000000..b83d4837dcb5
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.base.config;
+
+import lombok.Data;
+
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Data
+public class NettyRpcConfig {
+
+ private NettySslConfig nettySslConfig;
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
index cc0aa04f68c9..8869b9c1debf 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
@@ -79,4 +79,6 @@ public class NettyServerConfig {
*/
private int listenPort;
+ @Builder.Default
+ private NettySslConfig nettySslConfig = new NettySslConfig();
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java
new file mode 100644
index 000000000000..2524712fb3a3
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.base.config;
+
+import lombok.Data;
+
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Data
+public class NettySslConfig {
+
+ public boolean enabled;
+
+ public String certFilePath;
+
+ public String keyFilePath;
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
index 9ebf802b1ed2..f9c010170e75 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
@@ -19,16 +19,20 @@
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.dolphinscheduler.extract.base.exception.RemoteException;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder;
import org.apache.dolphinscheduler.extract.base.utils.NettyUtils;
+import java.io.File;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import javax.net.ssl.SSLException;
+
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import io.netty.bootstrap.ServerBootstrap;
@@ -40,8 +44,9 @@
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.timeout.IdleStateHandler;
-
/**
* remoting netty server
*/
@@ -66,7 +71,18 @@ class NettyRemotingServer {
private final AtomicBoolean isStarted = new AtomicBoolean(false);
- NettyRemotingServer(final NettyServerConfig serverConfig) {
+ private SslContext sslContext = null;
+
+ public NettyRemotingServer(final NettyServerConfig serverConfig) {
+ NettySslConfig nettySslConfig = serverConfig.getNettySslConfig();
+ if (nettySslConfig.isEnabled()) {
+ try {
+ sslContext = SslContextBuilder.forServer(new File(nettySslConfig.getCertFilePath()),
+ new File(nettySslConfig.getKeyFilePath())).build();
+ } catch (SSLException e) {
+ throw new RuntimeException(e);
+ }
+ }
this.serverConfig = serverConfig;
this.serverName = serverConfig.getServerName();
this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor(
@@ -130,6 +146,10 @@ protected void initChannel(SocketChannel ch) {
* @param ch socket channel
*/
private void initNettyChannel(SocketChannel ch) {
+ NettySslConfig nettySslConfig = serverConfig.getNettySslConfig();
+ if (nettySslConfig.isEnabled()) {
+ ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc()));
+ }
ch.pipeline()
.addLast("encoder", new TransporterEncoder())
.addLast("decoder", new TransporterDecoder())
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index 9ff59e0cdb94..8c79367c28f8 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.server.master.config;
import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.server.master.cluster.loadbalancer.WorkerLoadBalancerConfigurationProperties;
@@ -79,6 +80,8 @@ public class MasterConfig implements Validator {
*/
private String masterRegistryPath;
+ private NettySslConfig nettySslConfig;
+
@Override
public boolean supports(Class> clazz) {
return MasterConfig.class.isAssignableFrom(clazz);
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index 964fca836ce6..dc19ce8e4297 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -121,6 +121,11 @@ master:
memory-usage-weight: 30
cpu-usage-weight: 30
task-thread-pool-usage-weight: 40
+ rpc:
+ ssl:
+ enabled: false
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
server:
port: 5679
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
index d901effbd1db..7841d6d4eaea 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.server.worker.config;
import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.extract.base.config.NettySslConfig;
import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
@@ -47,7 +48,7 @@ public class WorkerConfig implements Validator {
private int hostWeight = 100;
private WorkerServerLoadProtection serverLoadProtection = new WorkerServerLoadProtection();
private ConnectStrategyProperties registryDisconnectStrategy = new ConnectStrategyProperties();
-
+ private NettySslConfig nettySslConfig;
/**
* This field doesn't need to set at config file, it will be calculated by workerIp:listenPort
*/
diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml
index 5cac4c29e5ae..4bc55ce1ed87 100644
--- a/dolphinscheduler-worker/src/main/resources/application.yaml
+++ b/dolphinscheduler-worker/src/main/resources/application.yaml
@@ -68,6 +68,11 @@ worker:
auto-create-tenant-enabled: true
# If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.
default-tenant-enabled: false
+ rpc:
+ ssl:
+ enabled: false
+ cert-file-path: /path/cert.crt
+ key-file-path: /path/private.pem
server:
port: 1235