From 4ab065a5f1d340ce160d55245e2e3e42eff4d857 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 11 Oct 2024 22:08:41 +0800 Subject: [PATCH 01/72] tmp store --- .../dolphinscheduler/alert/AlertServer.java | 4 ++- .../alert/config/AlertConfig.java | 3 ++ .../src/main/resources/application.yaml | 5 +++ .../api/ApiApplicationServer.java | 4 +++ .../src/main/resources/application.yaml | 5 +++ .../base/client/NettyRemotingClient.java | 22 ++++++++++++ .../base/config/NettyClientConfig.java | 3 ++ .../base/config/NettyServerConfig.java | 5 +++ .../extract/base/config/NettySslConfig.java | 34 +++++++++++++++++++ .../base/server/NettyRemotingServer.java | 24 +++++++++++-- .../server/master/MasterServer.java | 7 +++- .../server/master/config/MasterConfig.java | 3 ++ .../src/main/resources/application.yaml | 5 +++ .../server/worker/WorkerServer.java | 6 +++- .../server/worker/config/WorkerConfig.java | 3 +- .../src/main/resources/application.yaml | 5 +++ 16 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ff2e98542629..6e3688e6cc75 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import javax.annotation.PostConstruct; @@ -40,7 +41,8 @@ @Slf4j @Import({CommonConfiguration.class, DaoConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class AlertServer { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java index 240f92b846f8..084bc34c48fb 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; @@ -47,6 +48,8 @@ public final class AlertConfig implements Validator { private String alertServerAddress; + private NettySslConfig nettySslConfig; + @Override public boolean supports(Class clazz) { return AlertConfig.class.isAssignableFrom(clazz); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 927cbc3c2ce6..7251d9becb18 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -75,6 +75,11 @@ alert: max-heartbeat-interval: 60s # The maximum number of alerts that can be processed in parallel sender-parallelism: 100 + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem registry: type: zookeeper diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index b072fe630878..c4c2acce3459 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; @@ -52,6 +53,9 @@ public class ApiApplicationServer { @Autowired private PluginDao pluginDao; + @Autowired + NettySslConfig nettySslConfig; + public static void main(String[] args) { ApiServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance()); diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 9b0e94d64451..8ce78518c571 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -164,6 +164,11 @@ api: # Close each active connection of socket server if python program not active after x milliseconds. Define value is # (0 = infinite), and socket server would never close even though no requests accept read-timeout: 0 + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem metrics: enabled: true diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index 4aea4d6dfe9c..fed57160d239 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.extract.base.IRpcResponse; import org.apache.dolphinscheduler.extract.base.SyncRequestDto; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.RemotingException; import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; @@ -33,6 +34,7 @@ import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; +import java.io.File; import java.net.InetSocketAddress; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -41,6 +43,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; +import javax.net.ssl.SSLException; + import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -52,6 +56,8 @@ import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.timeout.IdleStateHandler; @Slf4j @@ -70,8 +76,20 @@ public class NettyRemotingClient implements AutoCloseable { private final NettyClientHandler clientHandler; + private SslContext sslContext = null; + public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; + NettySslConfig nettySslConfig = clientConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + try { + sslContext = + SslContextBuilder.forClient().trustManager(new File(nettySslConfig.getCertFilePath())).build(); + } catch (SSLException e) { + throw new IllegalArgumentException("Initialize SslContext error, please check the cert-file", e); + } + } + ThreadFactory nettyClientThreadFactory = ThreadUtils.newDaemonThreadFactory("NettyClientThread-"); if (Epoll.isAvailable()) { this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), nettyClientThreadFactory); @@ -97,6 +115,10 @@ private void start() { @Override public void initChannel(SocketChannel ch) { + NettySslConfig nettySslConfig = clientConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + } ch.pipeline() .addLast("client-idle-handler", new IdleStateHandler( diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java index a00ff540f4a5..974f9c43468c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java @@ -76,4 +76,7 @@ public class NettyClientConfig { @Builder.Default private int defaultRpcTimeoutMillis = 10_000; + @Builder.Default + private NettySslConfig nettySslConfig = new NettySslConfig(); + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java index cc0aa04f68c9..d7051dc37a47 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java @@ -79,4 +79,9 @@ public class NettyServerConfig { */ private int listenPort; + /** + * nettySslConfig + */ + @Builder.Default + private NettySslConfig nettySslConfig = new NettySslConfig(); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java new file mode 100644 index 000000000000..2524712fb3a3 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.config; + +import lombok.Data; + +import org.springframework.context.annotation.Configuration; + +@Configuration +@Data +public class NettySslConfig { + + public boolean enabled; + + public String certFilePath; + + public String keyFilePath; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java index 9ebf802b1ed2..f9c010170e75 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java @@ -19,16 +19,20 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.RemoteException; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; +import java.io.File; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import javax.net.ssl.SSLException; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.ServerBootstrap; @@ -40,8 +44,9 @@ import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.timeout.IdleStateHandler; - /** * remoting netty server */ @@ -66,7 +71,18 @@ class NettyRemotingServer { private final AtomicBoolean isStarted = new AtomicBoolean(false); - NettyRemotingServer(final NettyServerConfig serverConfig) { + private SslContext sslContext = null; + + public NettyRemotingServer(final NettyServerConfig serverConfig) { + NettySslConfig nettySslConfig = serverConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + try { + sslContext = SslContextBuilder.forServer(new File(nettySslConfig.getCertFilePath()), + new File(nettySslConfig.getKeyFilePath())).build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } this.serverConfig = serverConfig; this.serverName = serverConfig.getServerName(); this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor( @@ -130,6 +146,10 @@ protected void initChannel(SocketChannel ch) { * @param ch socket channel */ private void initNettyChannel(SocketChannel ch) { + NettySslConfig nettySslConfig = serverConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc())); + } ch.pipeline() .addLast("encoder", new TransporterEncoder()) .addLast("decoder", new TransporterDecoder()) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 4cffd3e37224..953efba7c6fc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -60,7 +61,8 @@ ServiceConfiguration.class, CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class MasterServer implements IStoppable { @@ -94,6 +96,9 @@ public class MasterServer implements IStoppable { @Autowired private SystemEventBusFireWorker systemEventBusFireWorker; + @Autowired + NettySslConfig nettySslConfig; + public static void main(String[] args) { MasterServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index 9ff59e0cdb94..8c79367c28f8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.master.config; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.cluster.loadbalancer.WorkerLoadBalancerConfigurationProperties; @@ -79,6 +80,8 @@ public class MasterConfig implements Validator { */ private String masterRegistryPath; + private NettySslConfig nettySslConfig; + @Override public boolean supports(Class clazz) { return MasterConfig.class.isAssignableFrom(clazz); diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index 964fca836ce6..dc19ce8e4297 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -121,6 +121,11 @@ master: memory-usage-weight: 30 cpu-usage-weight: 30 task-thread-pool-usage-weight: 40 + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem server: port: 5679 diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 4618c5ad59b6..f833967de630 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -55,7 +56,8 @@ @Slf4j @Import({CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class WorkerServer implements IStoppable { @@ -71,6 +73,8 @@ public class WorkerServer implements IStoppable { @Autowired private MetricsProvider metricsProvider; + @Autowired + NettySslConfig nettySslConfig; /** * worker server startup, not use web service * diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index d901effbd1db..7841d6d4eaea 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.worker.config; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; @@ -47,7 +48,7 @@ public class WorkerConfig implements Validator { private int hostWeight = 100; private WorkerServerLoadProtection serverLoadProtection = new WorkerServerLoadProtection(); private ConnectStrategyProperties registryDisconnectStrategy = new ConnectStrategyProperties(); - + private NettySslConfig nettySslConfig; /** * This field doesn't need to set at config file, it will be calculated by workerIp:listenPort */ diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index 5cac4c29e5ae..4bc55ce1ed87 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -68,6 +68,11 @@ worker: auto-create-tenant-enabled: true # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. default-tenant-enabled: false + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem server: port: 1235 From 9d0515bb8e4fd231a54a8cd0f56a83f4e373d03c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 12 Oct 2024 09:20:08 +0800 Subject: [PATCH 02/72] format the code --- .../org/apache/dolphinscheduler/alert/config/AlertConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java index 084bc34c48fb..a5217ed9d510 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.alert.config; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.commons.lang3.StringUtils; @@ -26,7 +27,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; From 71a167a75ddcd7d143a1baa0d78f1438524f263e Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 12 Oct 2024 14:36:15 +0800 Subject: [PATCH 03/72] fix ut --- .../org/apache/dolphinscheduler/api/ApiApplicationServer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index c4c2acce3459..8341a1cbb59f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -45,7 +45,8 @@ CommonConfiguration.class, ServiceConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @ServletComponentScan @SpringBootApplication public class ApiApplicationServer { From d3dc34f59056c7b93ecb83b2638c7bfdc11988dd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 16 Oct 2024 15:37:11 +0800 Subject: [PATCH 04/72] add e2e gluster-test --- .../docker/gluster-test/api/application.yaml | 254 ++++++++++++++++++ .../docker/gluster-test/docker-compose.yml | 166 ++++++++++++ .../gluster-test/master/application.yaml | 168 ++++++++++++ .../docker/gluster-test/ssl/cert.crt | 41 +++ .../docker/gluster-test/ssl/private.pem | 45 ++++ .../gluster-test/worker/application.yaml | 99 +++++++ 6 files changed, 773 insertions(+) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml new file mode 100644 index 000000000000..8305faa6fd3c --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml @@ -0,0 +1,254 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + port: 12345 + servlet: + session: + timeout: 120m + context-path: /dolphinscheduler/ + compression: + enabled: true + mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml + jetty: + max-http-form-post-size: 5000000 + accesslog: + enabled: true + custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms' + +spring: + profiles: + active: postgresql + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB + messages: + basename: i18n/messages + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + quartz: + auto-startup: false + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + mvc: + pathmatch: + matching-strategy: ANT_PATH_MATCHER + static-path-pattern: /static/** + cloud.discovery.client.composite-indicator.enabled: false + +springdoc: + swagger-ui: + path: /swagger-ui.html + packages-to-scan: org.apache.dolphinscheduler.api + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 1s + max-sleep: 3s + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + +api: + audit-enable: false + # Traffic control, if you turn on this config, the maximum number of request/s will be limited. + # global max request number per second + # default tenant-level max request number + traffic-control: + global-switch: false + max-global-qps-rate: 300 + tenant-switch: false + default-tenant-qps-rate: 10 + #customize-tenant-qps-rate: + # eg. + #tenant1: 11 + #tenant2: 20 + python-gateway: + # Weather enable python gateway server or not. The default value is false. + enabled: false + # Authentication token for connection from python api to python gateway server. Should be changed the default value + # when you deploy in public network. + auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc + # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different + # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` + gateway-server-address: 0.0.0.0 + # The port of Python gateway server start. Define which port you could connect to Python gateway server from + # Python API side. + gateway-server-port: 25333 + # The address of Python callback client. + python-address: 127.0.0.1 + # The port of Python callback client. + python-port: 25334 + # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), + # and socket server would never close even though no requests accept + connect-timeout: 0 + # Close each active connection of socket server if python program not active after x milliseconds. Define value is + # (0 = infinite), and socket server would never close even though no requests accept + read-timeout: 0 + rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + +metrics: + enabled: true + +security: + authentication: + # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO) + type: PASSWORD + # IF you set type `LDAP`, below config will be effective + ldap: + # ldap server config + urls: ldap://ldap.forumsys.com:389/ + base-dn: dc=example,dc=com + username: cn=read-only-admin,dc=example,dc=com + password: password + user: + # admin userId when you use LDAP login + admin: read-only-admin + identity-attribute: uid + email-attribute: mail + # action when ldap user is not exist (supported types: CREATE,DENY) + not-exist-action: CREATE + ssl: + enable: false + # jks file absolute path && password + trust-store: "/ldapkeystore.jks" + trust-store-password: "password" + casdoor: + user: + admin: "" + oauth2: + enable: false + provider: + github: + authorizationUri: "" + redirectUri: "" + clientId: "" + clientSecret: "" + tokenUri: "" + userInfoUri: "" + callbackUrl: "" + iconUri: "" + provider: github + google: + authorizationUri: "" + redirectUri: "" + clientId: "" + clientSecret: "" + tokenUri: "" + userInfoUri: "" + callbackUrl: "" + iconUri: "" + provider: google +casdoor: + # Your Casdoor server url + endpoint: "" + client-id: "" + client-secret: "" + # The certificate may be multi-line, you can use `|-` for ease + certificate: "" + # Your organization name added in Casdoor + organization-name: "" + # Your application name added in Casdoor + application-name: "" + # Doplhinscheduler login url + redirect-url: "" + + +# Override by profile + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml new file mode 100644 index 000000000000..ada5261f20b1 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + dolphinscheduler-postgresql: + image: bitnami/postgresql:15.2.0 + ports: + - "5432:5432" + profiles: ["all", "schema"] + environment: + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - dolphinscheduler + + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:3.7.1 + profiles: ["all"] + environment: + ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - dolphinscheduler + + dolphinscheduler-schema-initializer: + image: ${HUB}/dolphinscheduler-tools:${TAG} + env_file: .env + profiles: ["schema"] + command: [ tools/bin/upgrade-schema.sh ] + depends_on: + dolphinscheduler-postgresql: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + networks: + - dolphinscheduler + + dolphinscheduler-api: + image: ${HUB}/dolphinscheduler-api:${TAG} + ports: + - "12345:12345" + - "25333:25333" + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + - ./api/application.yaml:/opt/dolphinscheduler/api/conf/application.yaml + - ./ssl/cert.crt:/path/cert.crt + - ./ssl/private.pm:/path/private.pm + networks: + - dolphinscheduler + + dolphinscheduler-alert: + image: ${HUB}/dolphinscheduler-alert-server:${TAG} + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler + + dolphinscheduler-master: + image: ${HUB}/dolphinscheduler-master:${TAG} + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - ./master/application.yaml:/opt/dolphinscheduler/master/conf/application.yaml + - ./ssl/cert.crt:/path/cert.crt + - ./ssl/private.pm:/path/private.pm + networks: + - dolphinscheduler + + dolphinscheduler-worker: + image: ${HUB}/dolphinscheduler-worker:${TAG} + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + - ./worker/application.yaml:/:/opt/dolphinscheduler/worker/conf/application.yaml + - ./ssl/cert.crt:/path/cert.crt + - ./ssl/private.pm:/path/private.pm + networks: + - dolphinscheduler + +networks: + dolphinscheduler: + driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml new file mode 100644 index 000000000000..979d54a02221 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml @@ -0,0 +1,168 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +spring: + profiles: + active: postgresql + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + quartz: + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.threadPool.threadPriority: 5 + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.threadPool.makeThreadsDaemons: true + org.quartz.threadPool.threadCount: 25 + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + cloud.discovery.client.composite-indicator.enabled: false + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 1s + max-sleep: 3s + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + +master: + listen-port: 5678 + # The number of workflow event bus fire worker used to fire the workflow event, default is 2*CPU core + 1. + # workflow-event-bus-fire-thread-count: 10 + # The number of threads used to execute sync logic task e.g. Switch/Condition, default is CPU core. + # master-sync-task-executor-thread-pool-size: 10 + # The number of threads used to execute async logic task e.g. Dependent/SubWorkflow default is CPU core. + # master-async-task-executor-thread-pool-size: 10 + max-heartbeat-interval: 10s + server-load-protection: + # If set true, will open master overload protection + enabled: true + # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + max-jvm-cpu-usage-percentage-thresholds: 0.7 + # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.7 + # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. + max-disk-usage-percentage-thresholds: 0.7 + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: stop + worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 1 + # master fetch command num + fetch-size: 10 + worker-load-balancer-configuration-properties: + # RANDOM, ROUND_ROBIN, FIXED_WEIGHTED_ROUND_ROBIN, DYNAMIC_WEIGHTED_ROUND_ROBIN + type: DYNAMIC_WEIGHTED_ROUND_ROBIN + # dynamic-weight-config-properties only used in DYNAMIC_WEIGHTED_ROUND_ROBIN, the weight of memory-usage, cpu-usage, task-thread-pool-usage should sum to 100. + dynamic-weight-config-properties: + memory-usage-weight: 30 + cpu-usage-weight: 30 + task-thread-pool-usage-weight: 40 + rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + +server: + port: 5679 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +metrics: + enabled: true + +# Override by profile + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt new file mode 100644 index 000000000000..8b26de2debbf --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-----BEGIN CERTIFICATE----- +MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC +MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu +ZzEgMB4GA1UECgwXYXBhY2hlIGRvbHBoaW5zY2hlZHVsZXIxFjAUBgNVBAsMDUlU +IERlcGFydG1lbnQxGTAXBgNVBAMMEGRvbHBoaW5zY2hlZHVsZXIxOjA4BgkqhkiG +9w0BCQEWK3VzZXJzLXN1YnNjcmliZUBkb2xwaGluc2NoZWR1bGVyLmFwYWNoZS5v +cmcwHhcNMjQxMDE2MDcyODQ0WhcNMjUxMDE2MDcyODQ0WjCBwjELMAkGA1UEBhMC +Q04xEDAOBgNVBAgMB2JlaWppbmcxEDAOBgNVBAcMB2JlaWppbmcxIDAeBgNVBAoM +F2FwYWNoZSBkb2xwaGluc2NoZWR1bGVyMRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50 +MRkwFwYDVQQDDBBkb2xwaGluc2NoZWR1bGVyMTowOAYJKoZIhvcNAQkBFit1c2Vy +cy1zdWJzY3JpYmVAZG9scGhpbnNjaGVkdWxlci5hcGFjaGUub3JnMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwr5KUI0ZzVcpOjazaZUDo+y+uEnt1qRC +cmnnNFyMQuVhX5gBHRk+As5e+E+M3TTLpA4LFti644NqN9eUmuqTzEUAfRzd2dfh +cEKFU3TlhpmsLQ7nN/gYe0KMmh9imV7WFV25NPUsiGgEKB5EEFQkq0Ii6A7GHDUD +Cmwcchkwbgr4wCUVJfAjzfSsq1WuNnGlCeIzv9Xv/drqQ2StWuoDav2YoPsMqcTU +yXnrWWUW/L7TLMHZSe7uNphul3BRXNx6c7J9k9QRTDuyJt0seRM7PgFK/9aVKXxo +BG+k2pGLw+Xoq4URNS09YXsBiqEDy5A+RBUivfyE6nGFblraa6I++wIDAQABMA0G +CSqGSIb3DQEBCwUAA4IBAQA7LHY1apCsncI+GcDqvfClWBqJLyCCzLnQ4Z+vLsvC +C0zB0MXpqjXXe/FXcU62DNIXr0IyOlZTApuM7sLZftVYAihGzdMxQuhhoa1yZPo4 +2O6RulbRpxkFBNyfvOAGdmRi+pEfdXX2o/lqzgMqR2vzG79aOUYbgwPXIis4hZOP +Zp1Asz2JojmIRXXYVz00upCIuPs31D/0wZim6RG9RNqToiFCFc+wKENN/fhVwaSC +mz3GKWnagQA7JR6Dz9dVqqPtkhOaSW6KajFU7rvhg/C0VH8i2devOgAW6/7X6e2x ++XHg8jQ6GaYqDdqFk4T9Zpa/aMHBrnB0aHrrTJVrPSei +-----END CERTIFICATE----- diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem new file mode 100644 index 000000000000..a28335c31a91 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6 +NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3 +15Sa6pPMRQB9HN3Z1+FwQoVTdOWGmawtDuc3+Bh7QoyaH2KZXtYVXbk09SyIaAQo +HkQQVCSrQiLoDsYcNQMKbBxyGTBuCvjAJRUl8CPN9KyrVa42caUJ4jO/1e/92upD +ZK1a6gNq/Zig+wypxNTJeetZZRb8vtMswdlJ7u42mG6XcFFc3Hpzsn2T1BFMO7Im +3Sx5Ezs+AUr/1pUpfGgEb6TakYvD5eirhRE1LT1hewGKoQPLkD5EFSK9/ITqcYVu +Wtproj77AgMBAAECggEAV2e+BgpWftW0gNbFTsmdgSkEbsVrtM8zrZN/vH4fS/UU +76lTMqK+NNc4Q928fRoxnjfZXU84Dl5QwbCGF8PaQy/jKMRYhGpC2jZZDdylGfUG +FJEgUPwOzN/RKGmWwPySPLO8qRqzeqOo3fK7nNKPRLhh32KmI0ndzAM6/zD0Dich +CzGO+liQbhKNoBUXlZXhJs9Bu9mvkL4yId1eks8RZAt/Opi2CoijhLhs5ncdGRR9 +oB+D8KSSyuNo3+QQBoKLz++snC6EI/NTSRM86wDroVnYVxL7mXzmQ2fWNoUmAsX4 +ugXUAmCCz1d66tAJst0l30qMykiIT/jQg4uT22ecpQKBgQDiwfBMK+t0kzcZKiNF +u7jFWNGIHnJ4EU8xdzSbaWujuE7zP0NvC0e4Ej0DiJ7S4OBhbBuSDTSl0BQUfpXZ +O/UlSNJvgvCCjGXHnvtsXzza5fqvxjhGIbO/XOhSZZhriG1YDbCtitvfP3xR85EV +Os7/2KfztR2bFH66+lj51zYn3QKBgQDb23L443avaBxD4v9DKtGIP8JS7TPm4s39 +MgluzOvupH4dWZs83RSbsBa3AkzDvU268gRjwZYLFRwld13B8sfMWEMxTVX3iliZ +58GH/4DB7ixoMLU2yJglIQ8gKrJs3nJyUh2ilU7j2HWGlRvXlMgLFrgR0udgXmnP +un9+4L7AtwKBgQCeGBDtSoGymSEFDnfhGmxMSwxrFq9APvEzz6SB5hIB53JD7Yua +VAcAstaPLCVvaRGHWtQuE77tDBGz6D2ZY1AbWpY84NHGcxRTBOjOOeg3xbft7Qk2 +/DBR7via2fJWppt1lLXliLabA6wtht9UKdhAUFFofJHTmYpx2jFGn2FGkQKBgAcc +qjNy6Dv0WjW3+hnqIK0PovGbyroFTDSMkdGLv3YQzCOW6nalkC0ZHSL+aA1M6l7J +xWqUsI3bOPXdTqMff2cU09eccNN5HjScUp5bnYaodbdJHBORmZpZ32ti9jKfX4Rj +M6I2wj6MmgCogiYPIgl5yqqI8AMmCrY1ijpIeuMxAoGAdeisS8UakUdn0lVE5NSp +stncEwnjggL5gzVAHSQ/vzSddk7mUxCWZu//GryEJeFS7Q4jhBE7z5Y6GceiBraC +6YCauAAExswgi2wpxW9dwOOY1+zKVmUIoklUJ9nIuYFSxY/pDf1rQWUOUa3x104W +FYIRNxNXs7dGOs7Ge4M01Uw= +-----END PRIVATE KEY----- diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml new file mode 100644 index 000000000000..8965b044dcbf --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +spring: + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + cloud.discovery.client.composite-indicator.enabled: false + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 1s + max-sleep: 3s + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + +worker: + # worker listener port + listen-port: 1234 + # worker execute thread number to limit task instances in parallel + exec-threads: 100 + # worker heartbeat interval + max-heartbeat-interval: 10s + # worker host weight to dispatch tasks, default value 100 + host-weight: 100 + server-load-protection: + # If set true, will open worker overload protection + enabled: true + # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. + max-jvm-cpu-usage-percentage-thresholds: 0.7 + # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.7 + # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. + max-disk-usage-percentage-thresholds: 0.7 + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: stop + task-execute-threads-full-policy: REJECT + tenant-config: + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. + auto-create-tenant-enabled: true + # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. + default-tenant-enabled: false + rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + +server: + port: 1235 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +metrics: + enabled: true From 006581aae75e465d5239af4af8489c49f483a8f5 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 16 Oct 2024 19:59:22 +0800 Subject: [PATCH 05/72] add SslShellTaskE2ETest --- .github/workflows/e2e.yml | 2 + .../e2e/cases/ssl/SslShellTaskE2ETest.java | 288 ++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..03ab9a5fa2a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java new file mode 100644 index 000000000000..c89864263d11 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.e2e.cases.tasks; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; +import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; +import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; +import org.apache.dolphinscheduler.e2e.pages.LoginPage; +import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; +import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; +import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; +import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; +import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; +import org.apache.dolphinscheduler.e2e.pages.security.UserPage; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junitpioneer.jupiter.DisableIfTestFails; + +@TestMethodOrder(MethodOrderer.MethodName.class) +@DolphinScheduler(composeFiles = "docker/gluster-test/docker-compose.yaml") +@DisableIfTestFails +public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { + + @BeforeAll + public static void setup() { + browser = WebDriverHolder.getWebDriver(); + + TenantPage tenantPage = new LoginPage(browser) + .login(adminUser) + .goToNav(SecurityPage.class) + .goToTab(TenantPage.class); + + if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { + tenantPage + .create(adminUser.getTenant()) + .goToNav(SecurityPage.class) + .goToTab(UserPage.class) + .update(adminUser); + } + + tenantPage + .goToNav(ProjectPage.class) + .createProjectUntilSuccess(projectName); + } + + @Test + void testRunShellTasks_SuccessCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "SuccessCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo hello world\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_WorkflowParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "WorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "tom") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_LocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsOverrideWorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "jerry") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_UsingResourceFile() { + String testFileName = "echo"; + new ResourcePage(browser) + .goToNav(ResourcePage.class) + .goToTab(FileManagePage.class) + .createFileUntilSuccess(testFileName, "echo 123"); + + final WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "UsingResourceFile"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("cat " + testFileName + ".sh") + .name(taskName) + .selectResource(testFileName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_FailedCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "FailedCase"; + String taskName = "ShellFailed"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo 'I am failed'\n exit1\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + +} From 46e9db393365cebaedbf40ae6a38a5b6bdab2223 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 11:06:55 +0800 Subject: [PATCH 06/72] add doc and fix package --- docs/docs/en/guide/installation/cluster.md | 55 +++++++++++++++++ docs/docs/zh/guide/installation/cluster.md | 59 +++++++++++++++++++ .../e2e/cases/ssl/SslShellTaskE2ETest.java | 4 +- .../api/application.yaml | 0 .../docker-compose.yml | 0 .../master/application.yaml | 0 .../{gluster-test => ssl-test}/ssl/cert.crt | 0 .../ssl/private.pem | 0 .../worker/application.yaml | 0 .../base/config/NettyServerConfig.java | 3 - 10 files changed, 116 insertions(+), 5 deletions(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/api/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/docker-compose.yml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/master/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/ssl/cert.crt (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/ssl/private.pem (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/worker/application.yaml (100%) diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index 5dfc65699f2b..ebd8e292a89f 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -8,6 +8,61 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment. +## Enable SSL (optional) +In cluster deployment, you can enable SSL authentication. Secure Sockets Layer, SSL, abbreviated as SSL, is a secure protocol that encrypts transmitted data to ensure that information is not eavesdropped or tampered with during transmission. In addition, it can authenticate servers and ensure data integrity. + +To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files. + +Step 1: Install OpenSSL + +Firstly, ensure that you have installed OpenSSL. In most Linux distributions, OpenSSL is usually pre installed. If not, you can install it using the following command: + +On Ubuntu/Debian: +```bash +sudo apt-get install openssl +``` + +On CentOS/RHEL: +```bash +sudo yum install openssl +``` +Step 2: Generate private key (private.pem) + +Open the terminal and run the following command to generate a private key: + +```bash +openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 +``` + +This command will generate a 2048 bit RSA private key and save it as a private.pem file. + +Step 3: Generate Certificate Signing Request (CSR) + +Before generating a certificate, you need to generate a Certificate Signing Request (CSR). Run the following command: + +```bash +openssl req -new -key private.pem -out request.csr +``` +This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate. + +Step 4: Generate a self signed certificate (cert.crt) + +Use CSR to generate self signed certificates. Run the following command: +```bash +openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt +``` +This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file. + +Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, and `dolphinscheduler-api` modules. +```yaml +rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem +``` +You need to change `enabled` to `true` and configure the file routing for `cert-file-path` and `key-file-path`. + ### Prerequisites and DolphinScheduler Startup Environment Preparations Distribute the installation package to each server of each cluster and perform all the steps in [pseudo-cluster deployment](pseudo-cluster.md) on each machine. diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md index 5ed407ca1860..70b8e98f3d56 100644 --- a/docs/docs/zh/guide/installation/cluster.md +++ b/docs/docs/zh/guide/installation/cluster.md @@ -8,6 +8,65 @@ 集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样，所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器，而集群部署(Cluster)需要针对多台机器，且两者“修改相关配置”步骤区别较大 +### 开启SSL（可选） +在集群部署中，你可以开启SSL认证。Secure Sockets Layer，缩写作 SSL，是一种安全协议，能够加密传输的数据，确保在数据传输过程中，信息不会被窃听或篡改，此外还可以对服务器进行身份验证以及保障数据的完整性。 + +开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。 + +步骤1：安装 OpenSSL + +首先，确保您已经安装了 OpenSSL。在大多数 Linux 发行版中，OpenSSL 通常已预装。如果没有，您可以通过以下命令安装它： + +在 Ubuntu/Debian 上： + +```bash +sudo apt-get install openssl +``` + +在 CentOS/RHEL 上： + +```bash +sudo yum install openssl +``` + +步骤 2：生成私钥（private.pem） + +打开终端并运行以下命令生成私钥： + +```bash +openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 +``` +此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。 + +步骤 3：生成证书签署请求（CSR） + +在生成证书之前，您需要生成一个证书签署请求（CSR）。运行以下命令： + +```bash +openssl req -new -key private.pem -out request.csr +``` +此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。 + +步骤 4：生成自签名证书（cert.crt） + +使用 CSR 来生成自签名证书。运行以下命令： + +```bash +openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt +``` +此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。 + +然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`模块中的`application.yaml`文件。 +```yaml +rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem +``` +您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。 + + ### 前置准备工作 && 准备 DolphinScheduler 启动环境 需要将安装包分发至每台集群的每台服务器上，并且需要在每台机器中进行配置执行[伪集群部署](pseudo-cluster.md)中的所有执行项 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index c89864263d11..60bdc990aab3 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.e2e.cases.tasks; +package org.apache.dolphinscheduler.e2e.cases.ssl; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +42,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/gluster-test/docker-compose.yaml") +@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yaml") @DisableIfTestFails public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java index d7051dc37a47..8869b9c1debf 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java @@ -79,9 +79,6 @@ public class NettyServerConfig { */ private int listenPort; - /** - * nettySslConfig - */ @Builder.Default private NettySslConfig nettySslConfig = new NettySslConfig(); } From 32d0edbaf2cc911c9f3747e2f1751d3d8fce00b2 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 11:13:10 +0800 Subject: [PATCH 07/72] format doc --- docs/docs/en/guide/installation/cluster.md | 9 +++++++++ docs/docs/zh/guide/installation/cluster.md | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index ebd8e292a89f..44eae98c2c99 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -9,6 +9,7 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment. ## Enable SSL (optional) + In cluster deployment, you can enable SSL authentication. Secure Sockets Layer, SSL, abbreviated as SSL, is a secure protocol that encrypts transmitted data to ensure that information is not eavesdropped or tampered with during transmission. In addition, it can authenticate servers and ensure data integrity. To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files. @@ -18,14 +19,17 @@ Step 1: Install OpenSSL Firstly, ensure that you have installed OpenSSL. In most Linux distributions, OpenSSL is usually pre installed. If not, you can install it using the following command: On Ubuntu/Debian: + ```bash sudo apt-get install openssl ``` On CentOS/RHEL: + ```bash sudo yum install openssl ``` + Step 2: Generate private key (private.pem) Open the terminal and run the following command to generate a private key: @@ -43,17 +47,21 @@ Before generating a certificate, you need to generate a Certificate Signing Requ ```bash openssl req -new -key private.pem -out request.csr ``` + This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate. Step 4: Generate a self signed certificate (cert.crt) Use CSR to generate self signed certificates. Run the following command: + ```bash openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt ``` + This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file. Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, and `dolphinscheduler-api` modules. + ```yaml rpc: ssl: @@ -61,6 +69,7 @@ rpc: cert-file-path: /path/cert.crt key-file-path: /path/private.pem ``` + You need to change `enabled` to `true` and configure the file routing for `cert-file-path` and `key-file-path`. ### Prerequisites and DolphinScheduler Startup Environment Preparations diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md index 70b8e98f3d56..9cdeca86ee5c 100644 --- a/docs/docs/zh/guide/installation/cluster.md +++ b/docs/docs/zh/guide/installation/cluster.md @@ -9,6 +9,7 @@ 集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样，所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器，而集群部署(Cluster)需要针对多台机器，且两者“修改相关配置”步骤区别较大 ### 开启SSL（可选） + 在集群部署中，你可以开启SSL认证。Secure Sockets Layer，缩写作 SSL，是一种安全协议，能够加密传输的数据，确保在数据传输过程中，信息不会被窃听或篡改，此外还可以对服务器进行身份验证以及保障数据的完整性。 开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。 @@ -36,6 +37,7 @@ sudo yum install openssl ```bash openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 ``` + 此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。 步骤 3：生成证书签署请求（CSR） @@ -45,6 +47,7 @@ openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 ```bash openssl req -new -key private.pem -out request.csr ``` + 此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。 步骤 4：生成自签名证书（cert.crt） @@ -54,9 +57,11 @@ openssl req -new -key private.pem -out request.csr ```bash openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt ``` + 此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。 然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`模块中的`application.yaml`文件。 + ```yaml rpc: ssl: @@ -64,8 +69,8 @@ rpc: cert-file-path: /path/cert.crt key-file-path: /path/private.pem ``` -您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。 +您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。 ### 前置准备工作 && 准备 DolphinScheduler 启动环境 From 0333f83fb537f980bb834fcb744245f89deac6dd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 12:20:27 +0800 Subject: [PATCH 08/72] change docker-compose name --- .../dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 60bdc990aab3..2c438ba66e4a 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -42,7 +42,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yaml") +@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") @DisableIfTestFails public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { From 332fdfd52cc3431847898c1830d8c26e31c6f2f9 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 14:58:23 +0800 Subject: [PATCH 09/72] remove unnecessary content --- .../test/resources/docker/ssl-test/ssl/cert.crt | 17 ----------------- .../resources/docker/ssl-test/ssl/private.pem | 17 ----------------- 2 files changed, 34 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt index 8b26de2debbf..132745e62440 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt @@ -1,20 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -----BEGIN CERTIFICATE----- MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem index a28335c31a91..c0aab46c4eae 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem @@ -1,20 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6 NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3 From 289dd809bba909fba8d29a58570e93bf1eb99e69 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 19:04:01 +0800 Subject: [PATCH 10/72] fix e2e-test --- .../docker/ssl-test/api/application.yaml | 2 +- .../docker/ssl-test/master/application.yaml | 2 +- .../test/resources/docker/ssl-test/ssl/cert.crt | 17 +++++++++++++++++ .../resources/docker/ssl-test/ssl/private.pem | 17 +++++++++++++++++ .../docker/ssl-test/worker/application.yaml | 2 +- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml index 8305faa6fd3c..8ce78518c571 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml @@ -166,7 +166,7 @@ api: read-timeout: 0 rpc: ssl: - enabled: true + enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml index 979d54a02221..dc19ce8e4297 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml @@ -123,7 +123,7 @@ master: task-thread-pool-usage-weight: 40 rpc: ssl: - enabled: true + enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt index 132745e62440..12910f535382 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + -----BEGIN CERTIFICATE----- MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem index c0aab46c4eae..ba90f817b5c7 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6 NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml index 8965b044dcbf..4bc55ce1ed87 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml @@ -70,7 +70,7 @@ worker: default-tenant-enabled: false rpc: ssl: - enabled: true + enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem From a23a59cc81acc6249f08ab3bbf814aa1beac28e6 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 19:44:33 +0800 Subject: [PATCH 11/72] change e2e-test workflow name ad task name --- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 2c438ba66e4a..4745af29bd81 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -77,8 +77,8 @@ void testRunShellTasks_SuccessCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "SuccessCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslSuccessCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -112,8 +112,8 @@ void testRunShellTasks_WorkflowParamsCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "WorkflowParamsCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslWorkflowParamsCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -147,8 +147,8 @@ void testRunShellTasks_LocalParamsCase() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "LocalParamsCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslLocalParamsCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -182,8 +182,8 @@ void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "LocalParamsOverrideWorkflowParamsCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslLocalParamsOverrideWorkflowParamsCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -224,8 +224,8 @@ void testRunShellTasks_UsingResourceFile() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "UsingResourceFile"; - String taskName = "ShellSuccess"; + String workflowName = "SslUsingResourceFile"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -259,8 +259,8 @@ void testRunShellTasks_FailedCase() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "FailedCase"; - String taskName = "ShellFailed"; + String workflowName = "SslFailedCase"; + String taskName = "SslShellFailed"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) From be8df431a5abf5cb2b8bc6873fdce54bfc813d9b Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 19:57:07 +0800 Subject: [PATCH 12/72] delete unnecessary functions --- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 184 ------------------ 1 file changed, 184 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 4745af29bd81..b397f93ac839 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -29,8 +29,6 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; -import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; -import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; @@ -103,186 +101,4 @@ void testRunShellTasks_SuccessCase() { assertThat(taskInstance.retryTimes()).isEqualTo(0); } - @Test - void testRunShellTasks_WorkflowParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - // todo: use yaml to define the workflow - String workflowName = "SslWorkflowParamsCase"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "tom") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_LocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslLocalParamsCase"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslLocalParamsOverrideWorkflowParamsCase"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "jerry") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_UsingResourceFile() { - String testFileName = "echo"; - new ResourcePage(browser) - .goToNav(ResourcePage.class) - .goToTab(FileManagePage.class) - .createFileUntilSuccess(testFileName, "echo 123"); - - final WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslUsingResourceFile"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("cat " + testFileName + ".sh") - .name(taskName) - .selectResource(testFileName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_FailedCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslFailedCase"; - String taskName = "SslShellFailed"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("echo 'I am failed'\n exit1\n") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - } From a4ed1d04c3c9afd71150a6aa497bba9f8e81f3f0 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 20:22:07 +0800 Subject: [PATCH 13/72] change SslPythonTaskE2ETest --- .github/workflows/e2e.yml | 4 +-- ...E2ETest.java => SslPythonTaskE2ETest.java} | 33 ++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/{SslShellTaskE2ETest.java => SslPythonTaskE2ETest.java} (76%) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03ab9a5fa2a3..a7ab7778db85 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + - name: SslPythonTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslPythonTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java similarity index 76% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java index b397f93ac839..43eea2d79e78 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java @@ -22,27 +22,29 @@ import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; +import org.apache.dolphinscheduler.e2e.models.environment.PythonEnvironment; import org.apache.dolphinscheduler.e2e.pages.LoginPage; import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm; +import org.apache.dolphinscheduler.e2e.pages.security.EnvironmentPage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.junitpioneer.jupiter.DisableIfTestFails; -@TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") +@DolphinScheduler(composeFiles = "docker/ssl-task/docker-compose.yml") @DisableIfTestFails -public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { +public class SslPythonTaskE2ETest extends BaseWorkflowE2ETest { + + private static final PythonEnvironment pythonEnvironment = new PythonEnvironment(); @BeforeAll public static void setup() { @@ -60,6 +62,13 @@ public static void setup() { .goToTab(UserPage.class) .update(adminUser); } + tenantPage + .goToNav(SecurityPage.class) + .goToTab(EnvironmentPage.class) + .createEnvironmentUntilSuccess(pythonEnvironment.getEnvironmentName(), + pythonEnvironment.getEnvironmentConfig(), + pythonEnvironment.getEnvironmentDesc(), + pythonEnvironment.getEnvironmentWorkerGroup()); tenantPage .goToNav(ProjectPage.class) @@ -67,7 +76,8 @@ public static void setup() { } @Test - void testRunShellTasks_SuccessCase() { + @Order(10) + void testRunPythonTasks_SuccessCase() { WorkflowDefinitionTab workflowDefinitionPage = new ProjectPage(browser) .goToNav(ProjectPage.class) @@ -75,12 +85,13 @@ void testRunShellTasks_SuccessCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "SslSuccessCase"; - String taskName = "SslShellSuccess"; + String workflowName = "SslPythonSuccessCase"; + String taskName = "SslPythonSuccessTask"; + String pythonScripts = "print(\"success\")"; workflowDefinitionPage .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("echo hello world\n") + .addTask(WorkflowForm.TaskType.PYTHON) + .script(pythonScripts) .name(taskName) .submit() From a9638b42362ac8034cfa3531254f9afbb6cfc356 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 09:37:57 +0800 Subject: [PATCH 14/72] change SslShellTaskE2ETest --- .github/workflows/e2e.yml | 4 +- .../e2e/cases/ssl/SslPythonTaskE2ETest.java | 115 ------- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 288 ++++++++++++++++++ .../docker/ssl-test/docker-compose.yml | 10 +- 4 files changed, 295 insertions(+), 122 deletions(-) delete mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a7ab7778db85..03ab9a5fa2a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslPythonTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslPythonTaskE2ETest + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java deleted file mode 100644 index 43eea2d79e78..000000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.e2e.cases.ssl; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; -import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; -import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; -import org.apache.dolphinscheduler.e2e.models.environment.PythonEnvironment; -import org.apache.dolphinscheduler.e2e.pages.LoginPage; -import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm; -import org.apache.dolphinscheduler.e2e.pages.security.EnvironmentPage; -import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; -import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; -import org.apache.dolphinscheduler.e2e.pages.security.UserPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.DisableIfTestFails; - -@DolphinScheduler(composeFiles = "docker/ssl-task/docker-compose.yml") -@DisableIfTestFails -public class SslPythonTaskE2ETest extends BaseWorkflowE2ETest { - - private static final PythonEnvironment pythonEnvironment = new PythonEnvironment(); - - @BeforeAll - public static void setup() { - browser = WebDriverHolder.getWebDriver(); - - TenantPage tenantPage = new LoginPage(browser) - .login(adminUser) - .goToNav(SecurityPage.class) - .goToTab(TenantPage.class); - - if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { - tenantPage - .create(adminUser.getTenant()) - .goToNav(SecurityPage.class) - .goToTab(UserPage.class) - .update(adminUser); - } - tenantPage - .goToNav(SecurityPage.class) - .goToTab(EnvironmentPage.class) - .createEnvironmentUntilSuccess(pythonEnvironment.getEnvironmentName(), - pythonEnvironment.getEnvironmentConfig(), - pythonEnvironment.getEnvironmentDesc(), - pythonEnvironment.getEnvironmentWorkerGroup()); - - tenantPage - .goToNav(ProjectPage.class) - .createProjectUntilSuccess(projectName); - } - - @Test - @Order(10) - void testRunPythonTasks_SuccessCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - // todo: use yaml to define the workflow - String workflowName = "SslPythonSuccessCase"; - String taskName = "SslPythonSuccessTask"; - String pythonScripts = "print(\"success\")"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.PYTHON) - .script(pythonScripts) - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - -} diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java new file mode 100644 index 000000000000..2c438ba66e4a --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.e2e.cases.ssl; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; +import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; +import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; +import org.apache.dolphinscheduler.e2e.pages.LoginPage; +import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; +import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; +import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; +import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; +import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; +import org.apache.dolphinscheduler.e2e.pages.security.UserPage; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junitpioneer.jupiter.DisableIfTestFails; + +@TestMethodOrder(MethodOrderer.MethodName.class) +@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") +@DisableIfTestFails +public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { + + @BeforeAll + public static void setup() { + browser = WebDriverHolder.getWebDriver(); + + TenantPage tenantPage = new LoginPage(browser) + .login(adminUser) + .goToNav(SecurityPage.class) + .goToTab(TenantPage.class); + + if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { + tenantPage + .create(adminUser.getTenant()) + .goToNav(SecurityPage.class) + .goToTab(UserPage.class) + .update(adminUser); + } + + tenantPage + .goToNav(ProjectPage.class) + .createProjectUntilSuccess(projectName); + } + + @Test + void testRunShellTasks_SuccessCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "SuccessCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo hello world\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_WorkflowParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "WorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "tom") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_LocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsOverrideWorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "jerry") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_UsingResourceFile() { + String testFileName = "echo"; + new ResourcePage(browser) + .goToNav(ResourcePage.class) + .goToTab(FileManagePage.class) + .createFileUntilSuccess(testFileName, "echo 123"); + + final WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "UsingResourceFile"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("cat " + testFileName + ".sh") + .name(taskName) + .selectResource(testFileName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_FailedCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "FailedCase"; + String taskName = "ShellFailed"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo 'I am failed'\n exit1\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + +} diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml index ada5261f20b1..abacd8fd35f0 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml @@ -53,7 +53,7 @@ services: - dolphinscheduler dolphinscheduler-schema-initializer: - image: ${HUB}/dolphinscheduler-tools:${TAG} + image: ${HUB}/dolphinscheduler-tools:ci env_file: .env profiles: ["schema"] command: [ tools/bin/upgrade-schema.sh ] @@ -68,7 +68,7 @@ services: - dolphinscheduler dolphinscheduler-api: - image: ${HUB}/dolphinscheduler-api:${TAG} + image: ${HUB}/dolphinscheduler-api:ci ports: - "12345:12345" - "25333:25333" @@ -93,7 +93,7 @@ services: - dolphinscheduler dolphinscheduler-alert: - image: ${HUB}/dolphinscheduler-alert-server:${TAG} + image: ${HUB}/dolphinscheduler-alert-server:ci profiles: ["all"] env_file: .env healthcheck: @@ -110,7 +110,7 @@ services: - dolphinscheduler dolphinscheduler-master: - image: ${HUB}/dolphinscheduler-master:${TAG} + image: ${HUB}/dolphinscheduler-master:ci profiles: ["all"] env_file: .env healthcheck: @@ -131,7 +131,7 @@ services: - dolphinscheduler dolphinscheduler-worker: - image: ${HUB}/dolphinscheduler-worker:${TAG} + image: ${HUB}/dolphinscheduler-worker:ci profiles: ["all"] env_file: .env healthcheck: From 60ed52072b5683dad1b15e59ce1fb134b743efef Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 09:43:58 +0800 Subject: [PATCH 15/72] change ssl-test docker-compose --- .../test/resources/docker/ssl-test/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml index abacd8fd35f0..1ff331ee68c2 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml @@ -53,7 +53,7 @@ services: - dolphinscheduler dolphinscheduler-schema-initializer: - image: ${HUB}/dolphinscheduler-tools:ci + image: apache/dolphinscheduler-tools:ci env_file: .env profiles: ["schema"] command: [ tools/bin/upgrade-schema.sh ] @@ -68,7 +68,7 @@ services: - dolphinscheduler dolphinscheduler-api: - image: ${HUB}/dolphinscheduler-api:ci + image: apache/dolphinscheduler-api:ci ports: - "12345:12345" - "25333:25333" @@ -93,7 +93,7 @@ services: - dolphinscheduler dolphinscheduler-alert: - image: ${HUB}/dolphinscheduler-alert-server:ci + image: apache/dolphinscheduler-alert-server:ci profiles: ["all"] env_file: .env healthcheck: @@ -110,7 +110,7 @@ services: - dolphinscheduler dolphinscheduler-master: - image: ${HUB}/dolphinscheduler-master:ci + image: apache/dolphinscheduler-master:ci profiles: ["all"] env_file: .env healthcheck: @@ -131,7 +131,7 @@ services: - dolphinscheduler dolphinscheduler-worker: - image: ${HUB}/dolphinscheduler-worker:ci + image: apache/dolphinscheduler-worker:ci profiles: ["all"] env_file: .env healthcheck: From 7a0e835edef4e36caf15d3eb48b59c2f02c9153a Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 10:51:59 +0800 Subject: [PATCH 16/72] chanage e2e-k8s.yml --- .github/workflows/e2e-k8s.yml | 6 ++++++ .github/workflows/e2e.yml | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 30715f1dc8ba..a4475147a1d1 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -50,6 +50,12 @@ jobs: if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest timeout-minutes: 40 + strategy: + matrix: + case: + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03ab9a5fa2a3..f94c3af5dbad 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,7 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: From 3bc797ead5b27a72973bc123f9ee6fd2903b60af Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 11:24:24 +0800 Subject: [PATCH 17/72] chanage --- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 188 +----------------- .../docker/ssl-test/api/application.yaml | 2 +- .../docker/ssl-test/master/application.yaml | 2 +- .../docker/ssl-test/worker/application.yaml | 2 +- 4 files changed, 5 insertions(+), 189 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 2c438ba66e4a..b397f93ac839 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -29,8 +29,6 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; -import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; -import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; @@ -77,8 +75,8 @@ void testRunShellTasks_SuccessCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "SuccessCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslSuccessCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -103,186 +101,4 @@ void testRunShellTasks_SuccessCase() { assertThat(taskInstance.retryTimes()).isEqualTo(0); } - @Test - void testRunShellTasks_WorkflowParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - // todo: use yaml to define the workflow - String workflowName = "WorkflowParamsCase"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "tom") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_LocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "LocalParamsCase"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "LocalParamsOverrideWorkflowParamsCase"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "jerry") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_UsingResourceFile() { - String testFileName = "echo"; - new ResourcePage(browser) - .goToNav(ResourcePage.class) - .goToTab(FileManagePage.class) - .createFileUntilSuccess(testFileName, "echo 123"); - - final WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "UsingResourceFile"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("cat " + testFileName + ".sh") - .name(taskName) - .selectResource(testFileName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_FailedCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "FailedCase"; - String taskName = "ShellFailed"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("echo 'I am failed'\n exit1\n") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - } diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml index 8ce78518c571..8305faa6fd3c 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml @@ -166,7 +166,7 @@ api: read-timeout: 0 rpc: ssl: - enabled: false + enabled: true cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml index dc19ce8e4297..979d54a02221 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml @@ -123,7 +123,7 @@ master: task-thread-pool-usage-weight: 40 rpc: ssl: - enabled: false + enabled: true cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml index 4bc55ce1ed87..8965b044dcbf 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml @@ -70,7 +70,7 @@ worker: default-tenant-enabled: false rpc: ssl: - enabled: false + enabled: true cert-file-path: /path/cert.crt key-file-path: /path/private.pem From 190546b18761f33cf03475cffbdea0554dc52899 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 16:44:58 +0800 Subject: [PATCH 18/72] fix doc and remove unnecessary modifications --- .github/workflows/e2e.yml | 1 - docs/docs/en/guide/installation/cluster.md | 27 ++++------------------ docs/docs/zh/guide/installation/cluster.md | 26 ++++----------------- 3 files changed, 10 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f94c3af5dbad..eb43d4059258 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,7 +143,6 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index 44eae98c2c99..301123153a71 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -10,27 +10,10 @@ Cluster deployment uses the same scripts and configuration files as [pseudo-clus ## Enable SSL (optional) -In cluster deployment, you can enable SSL authentication. Secure Sockets Layer, SSL, abbreviated as SSL, is a secure protocol that encrypts transmitted data to ensure that information is not eavesdropped or tampered with during transmission. In addition, it can authenticate servers and ensure data integrity. - +In cluster deployment, you can enable SSL for secure internal communication. The DolphinScheduler cluster can be configured to use secured communication with internal authentication of the nodes in the cluster. To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files. -Step 1: Install OpenSSL - -Firstly, ensure that you have installed OpenSSL. In most Linux distributions, OpenSSL is usually pre installed. If not, you can install it using the following command: - -On Ubuntu/Debian: - -```bash -sudo apt-get install openssl -``` - -On CentOS/RHEL: - -```bash -sudo yum install openssl -``` - -Step 2: Generate private key (private.pem) +Step 1: Generate private key (private.pem) Open the terminal and run the following command to generate a private key: @@ -40,7 +23,7 @@ openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 This command will generate a 2048 bit RSA private key and save it as a private.pem file. -Step 3: Generate Certificate Signing Request (CSR) +Step 2: Generate Certificate Signing Request (CSR) Before generating a certificate, you need to generate a Certificate Signing Request (CSR). Run the following command: @@ -50,7 +33,7 @@ openssl req -new -key private.pem -out request.csr This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate. -Step 4: Generate a self signed certificate (cert.crt) +Step 3: Generate a self signed certificate (cert.crt) Use CSR to generate self signed certificates. Run the following command: @@ -60,7 +43,7 @@ openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file. -Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, and `dolphinscheduler-api` modules. +Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, `dolphinscheduler-api` and `dolphinscheduler-alert-server` modules. ```yaml rpc: diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md index 9cdeca86ee5c..c5aed2e3f204 100644 --- a/docs/docs/zh/guide/installation/cluster.md +++ b/docs/docs/zh/guide/installation/cluster.md @@ -10,27 +10,11 @@ ### 开启SSL（可选） -在集群部署中，你可以开启SSL认证。Secure Sockets Layer，缩写作 SSL，是一种安全协议，能够加密传输的数据，确保在数据传输过程中，信息不会被窃听或篡改，此外还可以对服务器进行身份验证以及保障数据的完整性。 +在集群部署中，您可以启用SSL以实现安全的内部通信。DolphinScheduler集群可以配置为使用安全通信，并对集群中的节点进行内部身份验证。 开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。 -步骤1：安装 OpenSSL - -首先，确保您已经安装了 OpenSSL。在大多数 Linux 发行版中，OpenSSL 通常已预装。如果没有，您可以通过以下命令安装它： - -在 Ubuntu/Debian 上： - -```bash -sudo apt-get install openssl -``` - -在 CentOS/RHEL 上： - -```bash -sudo yum install openssl -``` - -步骤 2：生成私钥（private.pem） +步骤 1：生成私钥（private.pem） 打开终端并运行以下命令生成私钥： @@ -40,7 +24,7 @@ openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。 -步骤 3：生成证书签署请求（CSR） +步骤 2：生成证书签署请求（CSR） 在生成证书之前，您需要生成一个证书签署请求（CSR）。运行以下命令： @@ -50,7 +34,7 @@ openssl req -new -key private.pem -out request.csr 此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。 -步骤 4：生成自签名证书（cert.crt） +步骤 3：生成自签名证书（cert.crt） 使用 CSR 来生成自签名证书。运行以下命令： @@ -60,7 +44,7 @@ openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt 此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。 -然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`模块中的`application.yaml`文件。 +然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`、`dolphinscheduler-alert-server`模块中的`application.yaml`文件。 ```yaml rpc: From 9607b2dc19c02d260e2ff1379190936218a40b03 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 19 Oct 2024 11:50:21 +0800 Subject: [PATCH 19/72] fix --- .github/workflows/e2e-k8s.yml | 5 ---- .github/workflows/e2e.yml | 2 ++ .../extract/base/config/NettyRpcConfig.java | 29 +++++++++++++++++++ .../server/master/MasterServer.java | 7 +---- .../server/worker/WorkerServer.java | 6 +--- 5 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index a4475147a1d1..b8bc1b003e43 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -50,11 +50,6 @@ jobs: if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest timeout-minutes: 40 - strategy: - matrix: - case: - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..03ab9a5fa2a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java new file mode 100644 index 000000000000..b83d4837dcb5 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.config; + +import lombok.Data; + +import org.springframework.context.annotation.Configuration; + +@Configuration +@Data +public class NettyRpcConfig { + + private NettySslConfig nettySslConfig; +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 953efba7c6fc..4cffd3e37224 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -61,8 +60,7 @@ ServiceConfiguration.class, CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class, - NettySslConfig.class}) + RegistryConfiguration.class}) @SpringBootApplication public class MasterServer implements IStoppable { @@ -96,9 +94,6 @@ public class MasterServer implements IStoppable { @Autowired private SystemEventBusFireWorker systemEventBusFireWorker; - @Autowired - NettySslConfig nettySslConfig; - public static void main(String[] args) { MasterServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index f833967de630..4618c5ad59b6 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -56,8 +55,7 @@ @Slf4j @Import({CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class, - NettySslConfig.class}) + RegistryConfiguration.class}) @SpringBootApplication public class WorkerServer implements IStoppable { @@ -73,8 +71,6 @@ public class WorkerServer implements IStoppable { @Autowired private MetricsProvider metricsProvider; - @Autowired - NettySslConfig nettySslConfig; /** * worker server startup, not use web service * From 5fb575a08d3824a9a7c73f9713705814d9f46b7c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 10:15:57 +0800 Subject: [PATCH 20/72] add ssl-e2e.yml --- .github/actions/labeler/labeler.yml | 1 + .github/workflows/e2e.yml | 2 - .github/workflows/ssl-e2e.yml | 194 ++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ssl-e2e.yml diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml index 6bd9b6daf1e5..c082dfbd2baa 100644 --- a/.github/actions/labeler/labeler.yml +++ b/.github/actions/labeler/labeler.yml @@ -63,6 +63,7 @@ e2e: - any: ['.github/workflows/e2e.yml'] - any: ['.github/workflows/api-test.yml'] - any: ['.github/workflows/e2e-k8s.yml'] + - any: ['.github/workflows/ssl-e2e.yml'] test: - any: ['**/test/**/*'] diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03ab9a5fa2a3..eb43d4059258 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,6 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml new file mode 100644 index 000000000000..33af19845133 --- /dev/null +++ b/.github/workflows/ssl-e2e.yml @@ -0,0 +1,194 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +on: + pull_request: + push: + branches: + - dev + - '[0-9]+.[0-9]+.[0-9]+-prepare' + - '[0-9]+.[0-9]+.[0-9]+-release' + +name: E2E + +concurrency: + group: e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: write + +jobs: + paths-filter: + name: E2E-Path-Filter + runs-on: ubuntu-latest + outputs: + not-ignore: ${{ steps.filter.outputs.not-ignore }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + id: filter + with: + filters: | + not-ignore: + - '!(docs/**)' + build: + name: E2E-Build + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - uses: actions/checkout@v4 + with: + submodules: true + - name: Sanity Check + uses: ./.github/actions/sanity-check + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + - name: Build Image + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip=true \ + -Dspotless.skip=true \ + -Pdocker,staging -Ddocker.tag=ci + - name: Export Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/standalone-image.tar + done + - uses: actions/upload-artifact@v4 + name: Upload master Docker Images + with: + name: standalone-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload worker Docker Images + with: + name: standalone-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload api Docker Images + with: + name: standalone-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tool Docker Images + with: + name: standalone-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload alert-server Docker Images + with: + name: standalone-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + e2e: + name: ${{ matrix.case.name }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + env: + RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'adopt' + - name: Collect Workflow Telemetry + uses: ./.github/actions/workflow-telemetry-action + with: + comment_on_pr: false + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + - uses: actions/download-artifact@v4 + name: Download Docker Images + with: + name: standalone-image-e2e + path: /tmp + - name: Load Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker load -i /tmp/${component}-image.tar + done + - name: Run Test + run: | + ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ + -DfailIfNoTests=false \ + -Dtest=${{ matrix.case.class }} test + - uses: actions/upload-artifact@v4 + if: always() + name: Upload Recording + with: + name: recording-${{ matrix.case.name }} + path: ${{ env.RECORDING_PATH }} + retention-days: 1 + result: + name: E2E + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [ e2e, paths-filter ] + if: always() + steps: + - name: Status + run: | + if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then + echo "Skip E2E!" + exit 0 + fi + if [[ ${{ needs.e2e.result }} != 'success' ]]; then + echo "E2E Failed!" + exit -1 + fi From 3e29c45be15ee7cbaa752619871e5c250d51aaff Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 10:50:58 +0800 Subject: [PATCH 21/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 33af19845133..a1bccdda5a36 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -23,7 +23,7 @@ on: - '[0-9]+.[0-9]+.[0-9]+-prepare' - '[0-9]+.[0-9]+.[0-9]+-release' -name: E2E +name: SSL-E2E concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} From e01ee101ac78f3e27e5bea3d7ed56bc7c0987516 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 13:59:47 +0800 Subject: [PATCH 22/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index a1bccdda5a36..371d9d801dec 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -26,7 +26,7 @@ on: name: SSL-E2E concurrency: - group: e2e-${{ github.event.pull_request.number || github.ref }} + group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: From 6355226865aab6c4c523f6d563c05ace955c792a Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:12:55 +0800 Subject: [PATCH 23/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 371d9d801dec..e15b7b03adb0 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -34,7 +34,7 @@ permissions: jobs: paths-filter: - name: E2E-Path-Filter + name: SSL-E2E-Path-Filter runs-on: ubuntu-latest outputs: not-ignore: ${{ steps.filter.outputs.not-ignore }} @@ -47,7 +47,7 @@ jobs: not-ignore: - '!(docs/**)' build: - name: E2E-Build + name: SSL-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest @@ -123,7 +123,7 @@ jobs: path: /tmp/alert-server-image.tar retention-days: 1 e2e: - name: ${{ matrix.case.name }} + name: ssl-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest timeout-minutes: 30 @@ -176,7 +176,7 @@ jobs: path: ${{ env.RECORDING_PATH }} retention-days: 1 result: - name: E2E + name: ssl-E2E runs-on: ubuntu-latest timeout-minutes: 30 needs: [ e2e, paths-filter ] From 829a29d5e8ef5ba8bbecec2b0ce40de9c5919da8 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:17:50 +0800 Subject: [PATCH 24/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index e15b7b03adb0..43753af2fc47 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -122,7 +122,7 @@ jobs: name: standalone-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 - e2e: + ssl-e2e: name: ssl-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest @@ -179,7 +179,7 @@ jobs: name: ssl-E2E runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ e2e, paths-filter ] + needs: [ ssl-e2e, paths-filter ] if: always() steps: - name: Status From 93076429d6d5cb084f37fa2ebd38fa1077021170 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:28:11 +0800 Subject: [PATCH 25/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 43753af2fc47..0db40b34fe20 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -46,7 +46,7 @@ jobs: filters: | not-ignore: - '!(docs/**)' - build: + ssl-build: name: SSL-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} @@ -124,7 +124,7 @@ jobs: retention-days: 1 ssl-e2e: name: ssl-${{ matrix.case.name }} - needs: build + needs: ssl-build runs-on: ubuntu-latest timeout-minutes: 30 strategy: @@ -176,7 +176,7 @@ jobs: path: ${{ env.RECORDING_PATH }} retention-days: 1 result: - name: ssl-E2E + name: SSL-E2E runs-on: ubuntu-latest timeout-minutes: 30 needs: [ ssl-e2e, paths-filter ] @@ -185,10 +185,10 @@ jobs: - name: Status run: | if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip E2E!" + echo "Skip SSL-E2E!" exit 0 fi if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "E2E Failed!" + echo "SSL-E2E Failed!" exit -1 fi From 5a1b13da905395432708f445231f04ea4578f198 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:35:23 +0800 Subject: [PATCH 26/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 0db40b34fe20..607fdbeb5d5c 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -46,7 +46,7 @@ jobs: filters: | not-ignore: - '!(docs/**)' - ssl-build: + build: name: SSL-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} @@ -124,7 +124,7 @@ jobs: retention-days: 1 ssl-e2e: name: ssl-${{ matrix.case.name }} - needs: ssl-build + needs: build runs-on: ubuntu-latest timeout-minutes: 30 strategy: From 5890d8adbc1718a24d717926ea656ed143aff54d Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:45:47 +0800 Subject: [PATCH 27/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 607fdbeb5d5c..a313176d6e09 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -129,6 +129,7 @@ jobs: timeout-minutes: 30 strategy: matrix: + case: - name: SslShellTaskE2ETest class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: From d79514ec2619daf4e8e0ddce905564b850585259 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 15:22:18 +0800 Subject: [PATCH 28/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 45 +++++-- ssl-e2e.yml | 222 ++++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 ssl-e2e.yml diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index a313176d6e09..078aeb630e14 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -90,36 +90,36 @@ jobs: components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/standalone-image.tar + && du -sh /tmp/${component}-image.tar done - uses: actions/upload-artifact@v4 name: Upload master Docker Images with: - name: standalone-image-e2e + name: master-image-e2e path: /tmp/master-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload worker Docker Images with: - name: standalone-image-e2e + name: worker-image-e2e path: /tmp/worker-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload api Docker Images with: - name: standalone-image-e2e + name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload tool Docker Images with: - name: standalone-image-e2e + name: tool-image-e2e path: /tmp/tool-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images with: - name: standalone-image-e2e + name: alert-server-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 ssl-e2e: @@ -153,11 +153,38 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- + + - uses: actions/download-artifact@v4 + name: download master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tool Docker Images + with: + name: tool-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 - uses: actions/download-artifact@v4 - name: Download Docker Images + name: download alert-server Docker Images with: - name: standalone-image-e2e - path: /tmp + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") diff --git a/ssl-e2e.yml b/ssl-e2e.yml new file mode 100644 index 000000000000..078aeb630e14 --- /dev/null +++ b/ssl-e2e.yml @@ -0,0 +1,222 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +on: + pull_request: + push: + branches: + - dev + - '[0-9]+.[0-9]+.[0-9]+-prepare' + - '[0-9]+.[0-9]+.[0-9]+-release' + +name: SSL-E2E + +concurrency: + group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: write + +jobs: + paths-filter: + name: SSL-E2E-Path-Filter + runs-on: ubuntu-latest + outputs: + not-ignore: ${{ steps.filter.outputs.not-ignore }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + id: filter + with: + filters: | + not-ignore: + - '!(docs/**)' + build: + name: SSL-E2E-Build + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - uses: actions/checkout@v4 + with: + submodules: true + - name: Sanity Check + uses: ./.github/actions/sanity-check + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + - name: Build Image + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip=true \ + -Dspotless.skip=true \ + -Pdocker,staging -Ddocker.tag=ci + - name: Export Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/${component}-image.tar + done + - uses: actions/upload-artifact@v4 + name: Upload master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tool Docker Images + with: + name: tool-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + ssl-e2e: + name: ssl-${{ matrix.case.name }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + case: + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + env: + RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'adopt' + - name: Collect Workflow Telemetry + uses: ./.github/actions/workflow-telemetry-action + with: + comment_on_pr: false + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + + - uses: actions/download-artifact@v4 + name: download master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tool Docker Images + with: + name: tool-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + + - name: Load Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker load -i /tmp/${component}-image.tar + done + - name: Run Test + run: | + ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ + -DfailIfNoTests=false \ + -Dtest=${{ matrix.case.class }} test + - uses: actions/upload-artifact@v4 + if: always() + name: Upload Recording + with: + name: recording-${{ matrix.case.name }} + path: ${{ env.RECORDING_PATH }} + retention-days: 1 + result: + name: SSL-E2E + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [ ssl-e2e, paths-filter ] + if: always() + steps: + - name: Status + run: | + if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then + echo "Skip SSL-E2E!" + exit 0 + fi + if [[ ${{ needs.e2e.result }} != 'success' ]]; then + echo "SSL-E2E Failed!" + exit -1 + fi From 614d58428060e125a7f56e4ced0da1c45be6b415 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 16:08:36 +0800 Subject: [PATCH 29/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 078aeb630e14..6bd8e5328ee5 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -81,11 +81,11 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: Build Image run: | - ./mvnw -B clean install \ + ./mvnw -B clean package \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Images + - name: Export Docker Imagesa run: | components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do From 452f51c8c23d04de2a1c65a47bda0fc469e3f8e2 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 17:06:35 +0800 Subject: [PATCH 30/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 6bd8e5328ee5..119bba16d2ac 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -85,11 +85,11 @@ jobs: -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Imagesa + - name: Export Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ + docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar done - uses: actions/upload-artifact@v4 From edbe7aa351237fae1bf4c41c313395342acd5695 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 19:00:03 +0800 Subject: [PATCH 31/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 119bba16d2ac..725d99766af6 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -114,7 +114,7 @@ jobs: name: Upload tool Docker Images with: name: tool-image-e2e - path: /tmp/tool-image.tar + path: /tmp/tools-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images @@ -176,7 +176,7 @@ jobs: name: download tool Docker Images with: name: tool-image-e2e - path: /tmp/tool-image.tar + path: /tmp/tools-image.tar retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images From e9038fd4e718c17437d4c468e15b8c5167ed2673 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 20:46:56 +0800 Subject: [PATCH 32/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 725d99766af6..9508cdfeba07 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -81,13 +81,13 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: Build Image run: | - ./mvnw -B clean package \ + ./mvnw -B clean package -o \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server") + components=("master" "worker" "api" "alert-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -110,12 +110,6 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tools-image.tar - retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images with: @@ -172,12 +166,6 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tools-image.tar - retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images with: @@ -187,7 +175,7 @@ jobs: - name: Load Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server") + components=("master" "worker" "api" "alert-server") for component in "${components[@]}"; do docker load -i /tmp/${component}-image.tar done From 6b4c53c017b6a08b370962b9bb0ff3ad596292fd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 08:59:47 +0800 Subject: [PATCH 33/72] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 9508cdfeba07..b218a6759543 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "alert-server") + components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -110,6 +110,12 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tools Docker Images + with: + name: tool-image-e2e + path: /tmp/tools-image.tar + retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images with: @@ -166,6 +172,12 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tools Docker Images + with: + name: tool-image-e2e + path: /tmp/tools-image.tar + retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images with: @@ -175,7 +187,7 @@ jobs: - name: Load Docker Images run: | - components=("master" "worker" "api" "alert-server") + components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do docker load -i /tmp/${component}-image.tar done From 852f155ab93c8fd515463baa75b558fc77386894 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 09:52:52 +0800 Subject: [PATCH 34/72] update ssl-e2e.yml --- .github/actions/labeler/labeler.yml | 2 +- .../{ssl-e2e.yml => cluster-e2e.yml} | 34 +++++++++---------- .../ClusterShellTaskE2ETest.java} | 4 +-- 3 files changed, 20 insertions(+), 20 deletions(-) rename .github/workflows/{ssl-e2e.yml => cluster-e2e.yml} (91%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/{ssl/SslShellTaskE2ETest.java => cluster/ClusterShellTaskE2ETest.java} (97%) diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml index c082dfbd2baa..3c951224920d 100644 --- a/.github/actions/labeler/labeler.yml +++ b/.github/actions/labeler/labeler.yml @@ -63,7 +63,7 @@ e2e: - any: ['.github/workflows/e2e.yml'] - any: ['.github/workflows/api-test.yml'] - any: ['.github/workflows/e2e-k8s.yml'] - - any: ['.github/workflows/ssl-e2e.yml'] + - any: ['.github/workflows/cluster-e2e.yml'] test: - any: ['**/test/**/*'] diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/cluster-e2e.yml similarity index 91% rename from .github/workflows/ssl-e2e.yml rename to .github/workflows/cluster-e2e.yml index b218a6759543..dafe2a8492ee 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -23,10 +23,10 @@ on: - '[0-9]+.[0-9]+.[0-9]+-prepare' - '[0-9]+.[0-9]+.[0-9]+-release' -name: SSL-E2E +name: CLUSTER-E2E concurrency: - group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} + group: cluster-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: @@ -34,7 +34,7 @@ permissions: jobs: paths-filter: - name: SSL-E2E-Path-Filter + name: CLUSTER-E2E-Path-Filter runs-on: ubuntu-latest outputs: not-ignore: ${{ steps.filter.outputs.not-ignore }} @@ -47,7 +47,7 @@ jobs: not-ignore: - '!(docs/**)' build: - name: SSL-E2E-Build + name: CLUSTER-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest @@ -122,16 +122,16 @@ jobs: name: alert-server-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 - ssl-e2e: - name: ssl-${{ matrix.case.name }} + cluster-e2e: + name: cluster-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: case: - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -158,31 +158,31 @@ jobs: name: download master Docker Images with: name: master-image-e2e - path: /tmp/master-image.tar + path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 name: download worker Docker Images with: name: worker-image-e2e - path: /tmp/worker-image.tar + path: /tmp retention-days: 1 - uses: actions/upload-artifact@v4 name: download api Docker Images with: name: api-image-e2e - path: /tmp/api-image.tar + path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 name: download tools Docker Images with: name: tool-image-e2e - path: /tmp/tools-image.tar + path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images with: name: alert-server-image-e2e - path: /tmp/alert-server-image.tar + path: /tmp retention-days: 1 - name: Load Docker Images @@ -204,19 +204,19 @@ jobs: path: ${{ env.RECORDING_PATH }} retention-days: 1 result: - name: SSL-E2E + name: CLUSTER-E2E runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ ssl-e2e, paths-filter ] + needs: [ cluster-e2e, paths-filter ] if: always() steps: - name: Status run: | if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip SSL-E2E!" + echo "Skip CLUSTER-E2E!" exit 0 fi if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "SSL-E2E Failed!" + echo "CLUSTER-E2E Failed!" exit -1 fi diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java similarity index 97% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index b397f93ac839..fd56561201f9 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.e2e.cases.ssl; +package org.apache.dolphinscheduler.e2e.cases.cluster; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +42,7 @@ @TestMethodOrder(MethodOrderer.MethodName.class) @DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") @DisableIfTestFails -public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { +public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { @BeforeAll public static void setup() { From 80cc92d8603ee46dd1229274f10558fc35fc685d Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 10:44:37 +0800 Subject: [PATCH 35/72] increase build time --- .github/workflows/cluster-e2e.yml | 4 +--- .github/workflows/e2e.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index dafe2a8492ee..7cc736b9dabe 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -153,7 +153,6 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v4 name: download master Docker Images with: @@ -175,7 +174,7 @@ jobs: - uses: actions/download-artifact@v4 name: download tools Docker Images with: - name: tool-image-e2e + name: tools-image-e2e path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 @@ -184,7 +183,6 @@ jobs: name: alert-server-image-e2e path: /tmp retention-days: 1 - - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..b63a0fa288d0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: From c24b79c5d333000db0fc17c103123f25a1b5afd4 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 11:13:40 +0800 Subject: [PATCH 36/72] update --- .github/workflows/cluster-e2e.yml | 34 +++++-------------------------- .github/workflows/e2e.yml | 2 +- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 7cc736b9dabe..a488a02f0964 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -130,8 +130,8 @@ jobs: strategy: matrix: case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -154,33 +154,9 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- - uses: actions/download-artifact@v4 - name: download master Docker Images + name: download cluster Docker Images with: - name: master-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download worker Docker Images - with: - name: worker-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: download api Docker Images - with: - name: api-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tools Docker Images - with: - name: tools-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download alert-server Docker Images - with: - name: alert-server-image-e2e + name: cluster-image-e2e path: /tmp retention-days: 1 - name: Load Docker Images @@ -217,4 +193,4 @@ jobs: if [[ ${{ needs.e2e.result }} != 'success' ]]; then echo "CLUSTER-E2E Failed!" exit -1 - fi + fi \ No newline at end of file diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b63a0fa288d0..eb43d4059258 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: From 37797c6fec5e0b70b4d9c56fc14e7c02dbc50691 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 14:15:37 +0800 Subject: [PATCH 37/72] update --- .github/workflows/cluster-e2e.yml | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index a488a02f0964..20ff80099949 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -130,8 +130,8 @@ jobs: strategy: matrix: case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -154,16 +154,40 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- - uses: actions/download-artifact@v4 - name: download cluster Docker Images + name: download master Docker Images with: - name: cluster-image-e2e - path: /tmp + name: master-image-e2e + path: /tmp/master + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tools Docker Images + with: + name: tools-image-e2e + path: /tmp/tools + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server retention-days: 1 - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do - docker load -i /tmp/${component}-image.tar + docker load -i /tmp/${component}/${component}-image.tar done - name: Run Test run: | From c85215b91c9104ecf288491480705c5cd80ab283 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 14:51:32 +0800 Subject: [PATCH 38/72] update --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 20ff80099949..0158b77cf25b 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -113,7 +113,7 @@ jobs: - uses: actions/upload-artifact@v4 name: Upload tools Docker Images with: - name: tool-image-e2e + name: tools-image-e2e path: /tmp/tools-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 From d53d0502ad6d05fb326f2371fa3251fa732a01fe Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 15:40:55 +0800 Subject: [PATCH 39/72] update --- .github/workflows/cluster-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 0158b77cf25b..987c93827041 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -171,6 +171,7 @@ jobs: name: api-image-e2e path: /tmp/api retention-days: 1 + overwrite: true - uses: actions/download-artifact@v4 name: download tools Docker Images with: From 734038ce5bc609f40d9813baa79e42d5e504af02 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 15:42:01 +0800 Subject: [PATCH 40/72] update --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 987c93827041..2c9d66429d28 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -165,7 +165,7 @@ jobs: name: worker-image-e2e path: /tmp/worker retention-days: 1 - - uses: actions/upload-artifact@v4 + - uses: actions/download-artifact@v4 name: download api Docker Images with: name: api-image-e2e From 5d836e90db35d469788d83906eef16333d4d0b9e Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 16:19:14 +0800 Subject: [PATCH 41/72] update --- .github/workflows/cluster-e2e.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 2c9d66429d28..65923db3a368 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -138,6 +138,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - name: Set up JDK 11 uses: actions/setup-java@v4 with: @@ -171,7 +181,6 @@ jobs: name: api-image-e2e path: /tmp/api retention-days: 1 - overwrite: true - uses: actions/download-artifact@v4 name: download tools Docker Images with: From e403cb014c8d6e20f7763ada50d75ee6e12f948a Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 23:25:45 +0800 Subject: [PATCH 42/72] update --- .github/workflows/cluster-e2e.yml | 20 +- .../cluster/ClusterShellTaskE2ETest.java | 2 +- .../api/application.yaml | 0 .../docker-compose.yml | 0 .../master/application.yaml | 0 .../{ssl-test => cluster-test}/ssl/cert.crt | 0 .../ssl/private.pem | 0 .../worker/application.yaml | 0 ssl-e2e.yml | 222 ------------------ 9 files changed, 6 insertions(+), 238 deletions(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/api/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/docker-compose.yml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/master/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/ssl/cert.crt (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/ssl/private.pem (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/worker/application.yaml (100%) delete mode 100644 ssl-e2e.yml diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 65923db3a368..703797fe0e78 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 40 steps: - uses: actions/checkout@v4 with: @@ -59,8 +59,8 @@ jobs: - name: Maximize runner space uses: ./.github/actions/maximize-build-space with: - root-reserve-mb: 30720 - temp-reserve-mb: 10240 + root-reserve-mb: 40720 + temp-reserve-mb: 20240 remove-dotnet: 'true' remove-android: 'true' remove-haskell: 'true' @@ -130,24 +130,14 @@ jobs: strategy: matrix: case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: - uses: actions/checkout@v4 with: submodules: true - - name: Maximize runner space - uses: ./.github/actions/maximize-build-space - with: - root-reserve-mb: 30720 - temp-reserve-mb: 10240 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' - name: Set up JDK 11 uses: actions/setup-java@v4 with: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index fd56561201f9..9e93b3ba3bfb 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -40,7 +40,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") +@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yml") @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml diff --git a/ssl-e2e.yml b/ssl-e2e.yml deleted file mode 100644 index 078aeb630e14..000000000000 --- a/ssl-e2e.yml +++ /dev/null @@ -1,222 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - - '[0-9]+.[0-9]+.[0-9]+-prepare' - - '[0-9]+.[0-9]+.[0-9]+-release' - -name: SSL-E2E - -concurrency: - group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - pull-requests: write - -jobs: - paths-filter: - name: SSL-E2E-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - build: - name: SSL-E2E-Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Maximize runner space - uses: ./.github/actions/maximize-build-space - with: - root-reserve-mb: 30720 - temp-reserve-mb: 10240 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' - - uses: actions/checkout@v4 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - name: Build Image - run: | - ./mvnw -B clean install \ - -Dmaven.test.skip=true \ - -Dspotless.skip=true \ - -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/${component}-image.tar - done - - uses: actions/upload-artifact@v4 - name: Upload master Docker Images - with: - name: master-image-e2e - path: /tmp/master-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload api Docker Images - with: - name: api-image-e2e - path: /tmp/api-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tool-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server-image.tar - retention-days: 1 - ssl-e2e: - name: ssl-${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - case: - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: 11 - distribution: 'adopt' - - name: Collect Workflow Telemetry - uses: ./.github/actions/workflow-telemetry-action - with: - comment_on_pr: false - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - - uses: actions/download-artifact@v4 - name: download master Docker Images - with: - name: master-image-e2e - path: /tmp/master-image.tar - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: download api Docker Images - with: - name: api-image-e2e - path: /tmp/api-image.tar - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tool-image.tar - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server-image.tar - retention-days: 1 - - - name: Load Docker Images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - docker load -i /tmp/${component}-image.tar - done - - name: Run Test - run: | - ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ - -DfailIfNoTests=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v4 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - result: - name: SSL-E2E - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ ssl-e2e, paths-filter ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip SSL-E2E!" - exit 0 - fi - if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "SSL-E2E Failed!" - exit -1 - fi From d642c1f5acc296c61c13849a0ee337e0c2b3bee6 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 23:30:32 +0800 Subject: [PATCH 43/72] update --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 703797fe0e78..bbdd7ddafd78 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -60,7 +60,7 @@ jobs: uses: ./.github/actions/maximize-build-space with: root-reserve-mb: 40720 - temp-reserve-mb: 20240 + temp-reserve-mb: 10240 remove-dotnet: 'true' remove-android: 'true' remove-haskell: 'true' From 4feeb56cc2e343a619e9cb92ee93d338828bd2ea Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 23 Oct 2024 17:49:26 +0800 Subject: [PATCH 44/72] update --- .github/actions/labeler/labeler.yml | 1 - .github/workflows/cluster-e2e.yml | 220 ------------------ .../cluster/ClusterShellTaskE2ETest.java | 2 +- .../docker/cluster-test/docker-compose.yaml | 85 +++++++ .../docker/cluster-test/docker-compose.yml | 166 ------------- 5 files changed, 86 insertions(+), 388 deletions(-) delete mode 100644 .github/workflows/cluster-e2e.yml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml delete mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml index 3c951224920d..6bd9b6daf1e5 100644 --- a/.github/actions/labeler/labeler.yml +++ b/.github/actions/labeler/labeler.yml @@ -63,7 +63,6 @@ e2e: - any: ['.github/workflows/e2e.yml'] - any: ['.github/workflows/api-test.yml'] - any: ['.github/workflows/e2e-k8s.yml'] - - any: ['.github/workflows/cluster-e2e.yml'] test: - any: ['**/test/**/*'] diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml deleted file mode 100644 index bbdd7ddafd78..000000000000 --- a/.github/workflows/cluster-e2e.yml +++ /dev/null @@ -1,220 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - - '[0-9]+.[0-9]+.[0-9]+-prepare' - - '[0-9]+.[0-9]+.[0-9]+-release' - -name: CLUSTER-E2E - -concurrency: - group: cluster-e2e-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - pull-requests: write - -jobs: - paths-filter: - name: CLUSTER-E2E-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - build: - name: CLUSTER-E2E-Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 40 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Maximize runner space - uses: ./.github/actions/maximize-build-space - with: - root-reserve-mb: 40720 - temp-reserve-mb: 10240 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' - - uses: actions/checkout@v4 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - name: Build Image - run: | - ./mvnw -B clean package -o \ - -Dmaven.test.skip=true \ - -Dspotless.skip=true \ - -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/${component}-image.tar - done - - uses: actions/upload-artifact@v4 - name: Upload master Docker Images - with: - name: master-image-e2e - path: /tmp/master-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload api Docker Images - with: - name: api-image-e2e - path: /tmp/api-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload tools Docker Images - with: - name: tools-image-e2e - path: /tmp/tools-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server-image.tar - retention-days: 1 - cluster-e2e: - name: cluster-${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: 11 - distribution: 'adopt' - - name: Collect Workflow Telemetry - uses: ./.github/actions/workflow-telemetry-action - with: - comment_on_pr: false - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v4 - name: download master Docker Images - with: - name: master-image-e2e - path: /tmp/master - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download api Docker Images - with: - name: api-image-e2e - path: /tmp/api - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tools Docker Images - with: - name: tools-image-e2e - path: /tmp/tools - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server - retention-days: 1 - - name: Load Docker Images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - docker load -i /tmp/${component}/${component}-image.tar - done - - name: Run Test - run: | - ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ - -DfailIfNoTests=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v4 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - result: - name: CLUSTER-E2E - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ cluster-e2e, paths-filter ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip CLUSTER-E2E!" - exit 0 - fi - if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "CLUSTER-E2E Failed!" - exit -1 - fi \ No newline at end of file diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index 9e93b3ba3bfb..728b01f4eb12 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -40,7 +40,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yml") +@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yaml") @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml new file mode 100644 index 000000000000..a1218ef94fa1 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:3.7.1 + profiles: [ "all" ] + environment: + ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181" ] + interval: 5s + timeout: 60s + retries: 120 + dolphinscheduler-api: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + dolphinscheduler-master: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "5678:5678" + - "5679:5679" + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + dolphinscheduler-worker: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "1234:1234" + - "1235:1235" + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +networks: + e2e: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml deleted file mode 100644 index 1ff331ee68c2..000000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml +++ /dev/null @@ -1,166 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3.8" - -services: - dolphinscheduler-postgresql: - image: bitnami/postgresql:15.2.0 - ports: - - "5432:5432" - profiles: ["all", "schema"] - environment: - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler - volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql - healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"] - interval: 5s - timeout: 60s - retries: 120 - networks: - - dolphinscheduler - - dolphinscheduler-zookeeper: - image: bitnami/zookeeper:3.7.1 - profiles: ["all"] - environment: - ALLOW_ANONYMOUS_LOGIN: "yes" - ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons - volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper - healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] - interval: 5s - timeout: 60s - retries: 120 - networks: - - dolphinscheduler - - dolphinscheduler-schema-initializer: - image: apache/dolphinscheduler-tools:ci - env_file: .env - profiles: ["schema"] - command: [ tools/bin/upgrade-schema.sh ] - depends_on: - dolphinscheduler-postgresql: - condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler - networks: - - dolphinscheduler - - dolphinscheduler-api: - image: apache/dolphinscheduler-api:ci - ports: - - "12345:12345" - - "25333:25333" - profiles: ["all"] - env_file: .env - healthcheck: - test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler - - ./api/application.yaml:/opt/dolphinscheduler/api/conf/application.yaml - - ./ssl/cert.crt:/path/cert.crt - - ./ssl/private.pm:/path/private.pm - networks: - - dolphinscheduler - - dolphinscheduler-alert: - image: apache/dolphinscheduler-alert-server:ci - profiles: ["all"] - env_file: .env - healthcheck: - test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: - - dolphinscheduler - - dolphinscheduler-master: - image: apache/dolphinscheduler-master:ci - profiles: ["all"] - env_file: .env - healthcheck: - test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - ./master/application.yaml:/opt/dolphinscheduler/master/conf/application.yaml - - ./ssl/cert.crt:/path/cert.crt - - ./ssl/private.pm:/path/private.pm - networks: - - dolphinscheduler - - dolphinscheduler-worker: - image: apache/dolphinscheduler-worker:ci - profiles: ["all"] - env_file: .env - healthcheck: - test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - volumes: - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler - - ./worker/application.yaml:/:/opt/dolphinscheduler/worker/conf/application.yaml - - ./ssl/cert.crt:/path/cert.crt - - ./ssl/private.pm:/path/private.pm - networks: - - dolphinscheduler - -networks: - dolphinscheduler: - driver: bridge - -volumes: - dolphinscheduler-postgresql: - dolphinscheduler-zookeeper: - dolphinscheduler-worker-data: - dolphinscheduler-logs: - dolphinscheduler-shared-local: - dolphinscheduler-resource-local: From 35d256c6605bf16a638a7dcbd59937073bcaf399 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 23 Oct 2024 17:52:05 +0800 Subject: [PATCH 45/72] update --- .github/workflows/e2e-k8s.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index b8bc1b003e43..30715f1dc8ba 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -50,7 +50,6 @@ jobs: if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest timeout-minutes: 40 - steps: - uses: actions/checkout@v4 with: From ba980d4b899e9b2c5bbf766377fce9b5a014606b Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 23 Oct 2024 18:50:21 +0800 Subject: [PATCH 46/72] update --- .../docker/cluster-test/docker-compose.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index a1218ef94fa1..7bd6526e5561 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,6 +31,11 @@ services: interval: 5s timeout: 60s retries: 120 + networks: + - e2e + deploy: + mode: replicated + replicas: 1 dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -41,6 +46,8 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy + networks: + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s @@ -58,6 +65,8 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy + networks: + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s @@ -75,11 +84,17 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy + networks: + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +volumes: + dolphinscheduler-zookeeper: + networks: e2e: + driver: bridge \ No newline at end of file From 1354ab6aec104c31c07602bc9268c6f5131c9ad8 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 23 Oct 2024 21:02:48 +0800 Subject: [PATCH 47/72] update --- .../src/test/resources/docker/cluster-test/docker-compose.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 7bd6526e5561..936407f12246 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -33,9 +33,6 @@ services: retries: 120 networks: - e2e - deploy: - mode: replicated - replicas: 1 dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: From 35ecb71b8ba714b5a9aa87cedba0caf8b0e88178 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 23 Oct 2024 21:07:42 +0800 Subject: [PATCH 48/72] update --- .github/workflows/e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..527482b5f789 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: From 88f851226127d498f7ee161050b63ecb1e948c35 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 23 Oct 2024 21:13:04 +0800 Subject: [PATCH 49/72] update --- .idea/vcs.xml | 6 +++++- .../test/resources/docker/cluster-test/docker-compose.yaml | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml index f30edcd8a1a3..e8be4ba2949f 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -12,5 +12,9 @@ + + + + - + \ No newline at end of file diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 936407f12246..f37c73ac85e3 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -42,7 +42,7 @@ services: - "12345:12345" depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully networks: - e2e healthcheck: @@ -61,7 +61,7 @@ services: - "5679:5679" depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully networks: - e2e healthcheck: @@ -80,7 +80,7 @@ services: - "1235:1235" depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully networks: - e2e healthcheck: From f91ab40da1d78519612d04878049589daf72630c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 24 Oct 2024 10:57:51 +0800 Subject: [PATCH 50/72] update --- .../docker/cluster-test/docker-compose.yaml | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index f37c73ac85e3..572edde9dc07 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -32,7 +32,8 @@ services: timeout: 60s retries: 120 networks: - - e2e + - dolphinscheduler + dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -42,15 +43,16 @@ services: - "12345:12345" depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy networks: - - e2e + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + dolphinscheduler-master: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -61,15 +63,16 @@ services: - "5679:5679" depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy networks: - - e2e + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + dolphinscheduler-worker: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -80,18 +83,24 @@ services: - "1235:1235" depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy networks: - - e2e + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] -volumes: - dolphinscheduler-zookeeper: networks: - e2e: - driver: bridge \ No newline at end of file + dolphinscheduler: + driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: \ No newline at end of file From 7839defc0b5fbcc224758d274c209ec395dd2579 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 24 Oct 2024 12:10:37 +0800 Subject: [PATCH 51/72] update --- .../docker/cluster-test/docker-compose.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 572edde9dc07..8d7f77c1c839 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -20,6 +20,8 @@ version: "3.8" services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 + ports: + - "2181:2181" profiles: [ "all" ] environment: ALLOW_ANONYMOUS_LOGIN: "yes" @@ -35,15 +37,14 @@ services: - dolphinscheduler dolphinscheduler-api: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-standalone-server:latest environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' ports: - "12345:12345" depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy + - dolphinscheduler-zookeeper networks: - dolphinscheduler healthcheck: @@ -54,7 +55,7 @@ services: command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-master: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-standalone-server:latest environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -62,8 +63,7 @@ services: - "5678:5678" - "5679:5679" depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy + - dolphinscheduler-zookeeper networks: - dolphinscheduler healthcheck: @@ -74,7 +74,7 @@ services: command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-standalone-server:latest environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -82,8 +82,7 @@ services: - "1234:1234" - "1235:1235" depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy + - dolphinscheduler-zookeeper networks: - dolphinscheduler healthcheck: From d0ca502c0d6a711d3d82e325d9de29f431c6a09c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 24 Oct 2024 20:17:46 +0800 Subject: [PATCH 52/72] update --- .../docker/cluster-test/docker-compose.yaml | 71 +++++++++++++------ 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 8d7f77c1c839..95059f8ce4c8 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -20,16 +20,13 @@ version: "3.8" services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 - ports: - - "2181:2181" - profiles: [ "all" ] environment: ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181" ] + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] interval: 5s timeout: 60s retries: 120 @@ -37,64 +34,92 @@ services: - dolphinscheduler dolphinscheduler-api: - image: apache/dolphinscheduler-standalone-server:latest + image: apache/dolphinscheduler-standalone-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' ports: - "12345:12345" - depends_on: - - dolphinscheduler-zookeeper + - "25333:25333" networks: - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 + depends_on: + dolphinscheduler-zookeeper: + condition: service_completed_successfully + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] - dolphinscheduler-master: - image: apache/dolphinscheduler-standalone-server:latest + dolphinscheduler-alert: + image: apache/dolphinscheduler-standalone-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "5678:5678" - - "5679:5679" + networks: + - dolphinscheduler + healthcheck: + test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] + interval: 5s + timeout: 5s + retries: 10 depends_on: - - dolphinscheduler-zookeeper + dolphinscheduler-zookeeper: + condition: service_completed_successfully + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] + + dolphinscheduler-master: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' networks: - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 + depends_on: + dolphinscheduler-zookeeper: + condition: service_completed_successfully + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:latest + image: apache/dolphinscheduler-standalone-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "1234:1234" - - "1235:1235" - depends_on: - - dolphinscheduler-zookeeper networks: - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 + depends_on: + dolphinscheduler-zookeeper: + condition: service_completed_successfully + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: dolphinscheduler: - driver: bridge volumes: dolphinscheduler-postgresql: From 136f2987beaa58f35a24908051e1caac34d53fe5 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 25 Oct 2024 17:17:36 +0800 Subject: [PATCH 53/72] update --- .../resources/docker/cluster-test/Dockerfile | 34 +++++++++++++++++++ .../docker/cluster-test/docker-compose.yaml | 32 ++++++++--------- 2 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile new file mode 100644 index 000000000000..21f9a4d70249 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jdk + +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +COPY apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 95059f8ce4c8..3df5721f77ce 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -1,10 +1,10 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# version: "3.8" @@ -32,9 +31,10 @@ services: retries: 120 networks: - dolphinscheduler +# command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-api:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -55,10 +55,10 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler - command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-alert-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -74,10 +74,10 @@ services: condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-master:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -94,10 +94,10 @@ services: volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-worker:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -116,7 +116,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler - command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: dolphinscheduler: From fb39e08ed12308f330eeabe90e51f7a45ece31ad Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 11:52:43 +0800 Subject: [PATCH 54/72] update --- .../src/test/resources/docker/cluster-test/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 21f9a4d70249..ace447834383 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,8 +27,8 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ - rm -rf apache-dolphinscheduler-*-bin + rm -rf apache-dolphinscheduler-*-bin \ No newline at end of file From fbfb5b95b065f3d9bea644dc241aebd9d1390199 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 13:25:44 +0800 Subject: [PATCH 55/72] update --- .../docker/cluster-test/{ => api}/Dockerfile | 6 ++- .../docker/cluster-test/docker-compose.yaml | 9 +++++ .../docker/cluster-test/master/Dockerfile | 38 +++++++++++++++++++ .../docker/cluster-test/worker/Dockerfile | 38 +++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/{ => api}/Dockerfile (90%) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile similarity index 90% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile index ace447834383..941a46e79a74 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile @@ -31,4 +31,8 @@ COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ - rm -rf apache-dolphinscheduler-*-bin \ No newline at end of file + rm -rf apache-dolphinscheduler-*-bin + +EXPOSE 12345 25333 + +CMD [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 3df5721f77ce..2d741b9853f8 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -34,6 +34,9 @@ services: # command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: + build: + context: . + dockerfile: ./api/Dockerfile image: apache/dolphinscheduler-api:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 @@ -77,6 +80,9 @@ services: # command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: + build: + context: . + dockerfile: ./master/Dockerfile image: apache/dolphinscheduler-master:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 @@ -97,6 +103,9 @@ services: # command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: + build: + context: . + dockerfile: ./worker/Dockerfile image: apache/dolphinscheduler-worker:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile new file mode 100644 index 000000000000..a89b6348b034 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jdk + +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin + +EXPOSE 12345 25333 + +CMD [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile new file mode 100644 index 000000000000..4f4dfdb7cc55 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jdk + +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin + +EXPOSE 12345 25333 + +CMD [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] From cd5055cb7922f06c7bcead738642e640bf102d1d Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 16:48:41 +0800 Subject: [PATCH 56/72] update --- .../docker/cluster-test/api/Dockerfile | 38 ------------------- .../docker/cluster-test/docker-compose.yaml | 19 +++------- .../docker/cluster-test/master/Dockerfile | 38 ------------------- .../docker/cluster-test/worker/Dockerfile | 38 ------------------- 4 files changed, 5 insertions(+), 128 deletions(-) delete mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile delete mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile delete mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile deleted file mode 100644 index 941a46e79a74..000000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jdk - -ENV DOCKER=true -ENV TZ=Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler - -RUN apt update ; \ - apt install -y sudo ; \ - rm -rf /var/lib/apt/lists/* - -WORKDIR $DOLPHINSCHEDULER_HOME - -COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME -RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ - rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ - mv apache-dolphinscheduler-*-bin/* . ; \ - rm -rf apache-dolphinscheduler-*-bin - -EXPOSE 12345 25333 - -CMD [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 2d741b9853f8..1739eabd10ac 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,12 +31,9 @@ services: retries: 120 networks: - dolphinscheduler -# command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] + command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: - build: - context: . - dockerfile: ./api/Dockerfile image: apache/dolphinscheduler-api:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 @@ -58,7 +55,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler -# command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: image: apache/dolphinscheduler-alert-server:ci @@ -77,12 +74,9 @@ services: condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs -# command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: - build: - context: . - dockerfile: ./master/Dockerfile image: apache/dolphinscheduler-master:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 @@ -100,12 +94,9 @@ services: volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft -# command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - build: - context: . - dockerfile: ./worker/Dockerfile image: apache/dolphinscheduler-worker:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 @@ -125,7 +116,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler -# command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: dolphinscheduler: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile deleted file mode 100644 index a89b6348b034..000000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jdk - -ENV DOCKER=true -ENV TZ=Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler - -RUN apt update ; \ - apt install -y sudo ; \ - rm -rf /var/lib/apt/lists/* - -WORKDIR $DOLPHINSCHEDULER_HOME - -COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME -RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ - rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ - mv apache-dolphinscheduler-*-bin/* . ; \ - rm -rf apache-dolphinscheduler-*-bin - -EXPOSE 12345 25333 - -CMD [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile deleted file mode 100644 index 4f4dfdb7cc55..000000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jdk - -ENV DOCKER=true -ENV TZ=Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler - -RUN apt update ; \ - apt install -y sudo ; \ - rm -rf /var/lib/apt/lists/* - -WORKDIR $DOLPHINSCHEDULER_HOME - -COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME -RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ - rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ - mv apache-dolphinscheduler-*-bin/* . ; \ - rm -rf apache-dolphinscheduler-*-bin - -EXPOSE 12345 25333 - -CMD [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] From e6806a3e9a4c426e841736844a2a5ef62dd695ec Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 19:37:29 +0800 Subject: [PATCH 57/72] update --- .../resources/docker/cluster-test/Dockerfile | 38 +++++++++++++++++++ .../docker/cluster-test/docker-compose.yaml | 12 ++++++ 2 files changed, 50 insertions(+) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile new file mode 100644 index 000000000000..d64546e5a3ec --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jdk + +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +COPY /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ + rm -rf /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ + mv /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin/* . ; \ + rm -rf /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin + +EXPOSE 12345 25333 + +CMD [ "/bin/bash", "/opt/dolphinscheduler/standalone-server/bin/start.sh" ] diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 1739eabd10ac..5d3369bb32de 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -35,6 +35,9 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler-api:ci + build: + context: . + dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -59,6 +62,9 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler-alert-server:ci + build: + context: . + dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -78,6 +84,9 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler-master:ci + build: + context: . + dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -98,6 +107,9 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler-worker:ci + build: + context: . + dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' From 0ccc213af6fd0f1709db4cb3d8007bcf292d2c4b Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 20:32:36 +0800 Subject: [PATCH 58/72] update --- .../src/test/resources/docker/cluster-test/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index d64546e5a3ec..50c72ed95892 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,11 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz $DOLPHINSCHEDULER_HOME -RUN tar -zxvf /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ - rm -rf /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ - mv /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin/* . ; \ - rm -rf /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin +COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ + rm -rf ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ + mv ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin/* . ; \ + rm -rf ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin EXPOSE 12345 25333 From d6797c9ab057b7b65d871174d024e64957cb58de Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 20:35:46 +0800 Subject: [PATCH 59/72] update --- .../src/test/resources/docker/cluster-test/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 50c72ed95892..0aef0359f98f 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,11 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz $DOLPHINSCHEDULER_HOME -RUN tar -zxvf ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ - rm -rf ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz ; \ - mv ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin/* . ; \ - rm -rf ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin +COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin EXPOSE 12345 25333 From 5b5046d5e56142059a26eb402ea8829fdc6c5567 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 20:37:34 +0800 Subject: [PATCH 60/72] update --- .../src/test/resources/docker/cluster-test/Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 0aef0359f98f..8bc7ccef6cbe 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -31,8 +31,4 @@ COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHI RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ - rm -rf apache-dolphinscheduler-*-bin - -EXPOSE 12345 25333 - -CMD [ "/bin/bash", "/opt/dolphinscheduler/standalone-server/bin/start.sh" ] + rm -rf apache-dolphinscheduler-*-bin \ No newline at end of file From ac36bc2cbf27cd515a1f7f87abae539d47183f18 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 21:39:03 +0800 Subject: [PATCH 61/72] update --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 527482b5f789..ac9a35dc0828 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -88,7 +88,7 @@ jobs: - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar + && cp /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /tmp/dist/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images with: From 800ee933d4478e027e5604242ac38d140a34d596 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 21:50:42 +0800 Subject: [PATCH 62/72] update --- .../src/main/docker/standalone-server.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile index 0abd547ea210..6257c2f7b4e3 100644 --- a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY /tmp/dist//apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ From 13ef350d1b153893af7f3c96030f6240d1216b1c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 23:13:48 +0800 Subject: [PATCH 63/72] update --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ac9a35dc0828..be20d408d8af 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -88,7 +88,7 @@ jobs: - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && cp /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /tmp/dist/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + && du -sh /tmp/standalone-image.tar && cp /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /tmp/dist/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images with: From 21a8025fb942ef470fb69bd3b87f54a6613ab947 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 23:15:50 +0800 Subject: [PATCH 64/72] update --- .../src/main/docker/standalone-server.dockerfile | 2 +- .../src/test/resources/docker/cluster-test/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile index 6257c2f7b4e3..0abd547ea210 100644 --- a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY /tmp/dist//apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 8bc7ccef6cbe..6e6ad79e4a1d 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY ./target/dolphinscheduler-dist/target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ From 2b08d2f1a35a54bc72bab06cd55ecd905f05f2a7 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 26 Oct 2024 23:17:53 +0800 Subject: [PATCH 65/72] update --- .../src/test/resources/docker/cluster-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 6e6ad79e4a1d..93947517416b 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY ./target/dolphinscheduler-dist/target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY /tmp/dist/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ From 8dcbf9f747ae5bbf5effdabe0ddc728719ea1edd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sun, 27 Oct 2024 00:10:12 +0800 Subject: [PATCH 66/72] update --- .github/workflows/e2e.yml | 76 +++++++++++++++++-- .../docker/cluster-test/docker-compose.yaml | 12 --- 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index be20d408d8af..fd569927b6e1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,14 +87,47 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar && cp /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /tmp/dist/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + components=("master" "worker" "api" "tools" "alert-server","standalone") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/${component}-image.tar + done - uses: actions/upload-artifact@v4 name: Upload Docker Images with: name: standalone-image-e2e path: /tmp/standalone-image.tar retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tools Docker Images + with: + name: tools-image-e2e + path: /tmp/tools-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 e2e: name: ${{ matrix.case.name }} needs: build @@ -170,10 +203,43 @@ jobs: name: Download Docker Images with: name: standalone-image-e2e - path: /tmp + path: /tmp/standalone + - uses: actions/download-artifact@v4 + name: download master Docker Images + with: + name: master-image-e2e + path: /tmp/master + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tools Docker Images + with: + name: tools-image-e2e + path: /tmp/tools + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server + retention-days: 1 - name: Load Docker Images run: | - docker load -i /tmp/standalone-image.tar + components=("master" "worker" "api" "tools" "alert-server","standalone") + for component in "${components[@]}"; do + docker load -i /tmp/${component}/${component}-image.tar + done - name: Run Test run: | ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ @@ -202,4 +268,4 @@ jobs: if [[ ${{ needs.e2e.result }} != 'success' ]]; then echo "E2E Failed!" exit -1 - fi + fi \ No newline at end of file diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 5d3369bb32de..1739eabd10ac 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -35,9 +35,6 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler-api:ci - build: - context: . - dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -62,9 +59,6 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler-alert-server:ci - build: - context: . - dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -84,9 +78,6 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler-master:ci - build: - context: . - dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -107,9 +98,6 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler-worker:ci - build: - context: . - dockerfile: Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' From d1bbd7e128fee6097a199775ddb57a9ab117d02b Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sun, 27 Oct 2024 09:19:39 +0800 Subject: [PATCH 67/72] update --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fd569927b6e1..06282282c378 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server","standalone") + components=("master" "worker" "api" "tools" "alert-server" "standalone") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -236,7 +236,7 @@ jobs: retention-days: 1 - name: Load Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server","standalone") + components=("master" "worker" "api" "tools" "alert-server" "standalone") for component in "${components[@]}"; do docker load -i /tmp/${component}/${component}-image.tar done From 7d798bb4b62b3f302b6c6e0c6d911319ffcc65ae Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 30 Oct 2024 19:12:02 +0800 Subject: [PATCH 68/72] update --- .github/workflows/e2e.yml | 23 +++++++++++++++---- .../resources/docker/cluster-test/Dockerfile | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 06282282c378..8b285db57d0f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server" "standalone") + components=("master" "worker" "api" "tools" "alert-server" "standalone-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -96,7 +96,7 @@ jobs: name: Upload Docker Images with: name: standalone-image-e2e - path: /tmp/standalone-image.tar + path: /tmp/standalone-server-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload master Docker Images @@ -128,6 +128,11 @@ jobs: name: alert-server-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload dist file + with: + name: bin-tar + path: /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz e2e: name: ${{ matrix.case.name }} needs: build @@ -203,7 +208,7 @@ jobs: name: Download Docker Images with: name: standalone-image-e2e - path: /tmp/standalone + path: /tmp/standalone-server - uses: actions/download-artifact@v4 name: download master Docker Images with: @@ -234,9 +239,17 @@ jobs: name: alert-server-image-e2e path: /tmp/alert-server retention-days: 1 + - uses: actions/download-artifact@v4 + name: Download dist file + with: + name: bin-tar + path: /tmp + - name: untar dist file + run: + tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz && pwd - name: Load Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server" "standalone") + components=("master" "worker" "api" "tools" "alert-server" "standalone-server") for component in "${components[@]}"; do docker load -i /tmp/${component}/${component}-image.tar done diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 93947517416b..55aa5392cc0c 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY /tmp/dist/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY /tmp/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ From c1c4386917d54900d642474e3c0dada694fe7d43 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 30 Oct 2024 20:06:58 +0800 Subject: [PATCH 69/72] update --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8b285db57d0f..90019b59999b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -132,7 +132,7 @@ jobs: name: Upload dist file with: name: bin-tar - path: /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + path: /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz e2e: name: ${{ matrix.case.name }} needs: build From c9e43520365b3a009c322b3b9af00c44291cafbc Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 31 Oct 2024 14:21:31 +0800 Subject: [PATCH 70/72] update --- .github/workflows/e2e.yml | 91 +++------------------------------------ 1 file changed, 5 insertions(+), 86 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 90019b59999b..22b7783e280c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: @@ -87,52 +87,14 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server" "standalone-server") - for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/${component}-image.tar - done + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ + && du -sh /tmp/standalone-image.tar - uses: actions/upload-artifact@v4 name: Upload Docker Images with: name: standalone-image-e2e - path: /tmp/standalone-server-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload master Docker Images - with: - name: master-image-e2e - path: /tmp/master-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload api Docker Images - with: - name: api-image-e2e - path: /tmp/api-image.tar + path: /tmp/standalone-image.tar retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload tools Docker Images - with: - name: tools-image-e2e - path: /tmp/tools-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload dist file - with: - name: bin-tar - path: /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz e2e: name: ${{ matrix.case.name }} needs: build @@ -181,8 +143,6 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -208,51 +168,10 @@ jobs: name: Download Docker Images with: name: standalone-image-e2e - path: /tmp/standalone-server - - uses: actions/download-artifact@v4 - name: download master Docker Images - with: - name: master-image-e2e - path: /tmp/master - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download api Docker Images - with: - name: api-image-e2e - path: /tmp/api - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tools Docker Images - with: - name: tools-image-e2e - path: /tmp/tools - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server - retention-days: 1 - - uses: actions/download-artifact@v4 - name: Download dist file - with: - name: bin-tar path: /tmp - - name: untar dist file - run: - tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz && pwd - name: Load Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server" "standalone-server") - for component in "${components[@]}"; do - docker load -i /tmp/${component}/${component}-image.tar - done + docker load -i /tmp/standalone-image.tar - name: Run Test run: | ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ From 876a4a43542ec5d95b6b50d0ba3fde064d4ff0bb Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 31 Oct 2024 14:46:57 +0800 Subject: [PATCH 71/72] update --- .github/workflows/e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 22b7783e280c..08061d20bc4b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: From 5961df63377893b1f9abc7772f91c18a92b6faa1 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 31 Oct 2024 20:49:15 +0800 Subject: [PATCH 72/72] update --- .github/workflows/e2e.yml | 2 -- .idea/vcs.xml | 4 ---- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 08061d20bc4b..22b7783e280c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,6 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/.idea/vcs.xml b/.idea/vcs.xml index e8be4ba2949f..ee2912d17902 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -12,9 +12,5 @@ - - - - \ No newline at end of file