From cb06faba99c89b3d072efd5ea9ad2c41763e879f Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 25 Sep 2024 18:24:54 +0800 Subject: [PATCH 1/5] add netty ssl --- .../dolphinscheduler/alert/AlertServer.java | 4 ++- .../alert/rpc/AlertRpcServer.java | 5 +-- .../src/main/resources/application.yaml | 6 ++++ .../alert/rpc/AlertRpcServerTest.java | 3 +- .../api/ApiApplicationServer.java | 4 +++ .../src/main/resources/application.yaml | 5 +++ .../api/service/LoggerServiceTest.java | 3 +- dolphinscheduler-bom/pom.xml | 4 +-- .../extract/base/client/Clients.java | 2 +- .../base/client/NettyRemotingClient.java | 21 ++++++++++- .../client/NettyRemotingClientFactory.java | 6 ++-- .../extract/base/config/NettySslConfig.java | 36 +++++++++++++++++++ .../base/server/NettyRemotingServer.java | 25 ++++++++++++- .../server/NettyRemotingServerFactory.java | 6 ++-- .../extract/base/server/RpcServer.java | 5 +-- .../SpringServerMethodInvokerDiscovery.java | 5 +-- .../extract/base/client/ClientsTest.java | 2 +- dolphinscheduler-master/pom.xml | 4 +++ .../server/master/MasterServer.java | 7 +++- .../server/master/rpc/MasterRpcServer.java | 5 +-- .../src/main/resources/application.yaml | 25 ++++++++----- .../src/main/resources/logback-spring.xml | 1 + .../master/rpc/MasterRpcServerTest.java | 3 +- .../microbench/rpc/RpcBenchMarkTest.java | 3 +- .../server/worker/WorkerServer.java | 7 +++- .../server/worker/rpc/WorkerRpcServer.java | 5 +-- .../src/main/resources/application.yaml | 6 ++++ .../worker/rpc/WorkerRpcServerTest.java | 3 +- 28 files changed, 174 insertions(+), 37 deletions(-) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ff2e98542629..6e3688e6cc75 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import javax.annotation.PostConstruct; @@ -40,7 +41,8 @@ @Slf4j @Import({CommonConfiguration.class, DaoConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class AlertServer { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java index d73e4755ddd4..aeaf100deb99 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import lombok.extern.slf4j.Slf4j; @@ -29,8 +30,8 @@ @Service public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { - public AlertRpcServer(AlertConfig alertConfig) { - super(NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build()); + public AlertRpcServer(AlertConfig alertConfig, NettySslConfig nettySslConfig) { + super(NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build(),nettySslConfig); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 927cbc3c2ce6..00c73bb759b1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -93,6 +93,12 @@ registry: metrics: enabled: true +rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + # Override by profile --- diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java index 75f16848fdf0..cdd905cbbf11 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java @@ -19,11 +19,12 @@ import org.apache.dolphinscheduler.alert.config.AlertConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.junit.jupiter.api.Test; class AlertRpcServerTest { - private final AlertRpcServer alertRpcServer = new AlertRpcServer(new AlertConfig()); + private final AlertRpcServer alertRpcServer = new AlertRpcServer(new AlertConfig(),new NettySslConfig()); @Test void testStart() { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index b072fe630878..c4c2acce3459 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; @@ -52,6 +53,9 @@ public class ApiApplicationServer { @Autowired private PluginDao pluginDao; + @Autowired + NettySslConfig nettySslConfig; + public static void main(String[] args) { ApiServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance()); diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 9b0e94d64451..f10aa2a9c757 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -231,6 +231,11 @@ casdoor: # Doplhinscheduler login url redirect-url: "" +rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem # Override by profile diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index c6ced818c6df..c20f25c54935 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -41,6 +41,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.extract.common.ILogService; import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdRequest; @@ -103,7 +104,7 @@ public void setUp() { } springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery( - NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build()); + NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build(),new NettySslConfig()); springServerMethodInvokerDiscovery.start(); springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new ILogService() { diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 2b14225d8e53..baccfb4d8e93 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -392,7 +392,7 @@ mysql mysql-connector-java ${mysql-connector.version} - test + @@ -946,7 +946,7 @@ org.testcontainers mysql ${testcontainer.version} - test + diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java index c4741e0a3b58..50ea4b589735 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java @@ -36,7 +36,7 @@ public class Clients { private static final JdkDynamicRpcClientProxyFactory jdkDynamicRpcClientProxyFactory = new JdkDynamicRpcClientProxyFactory( NettyRemotingClientFactory.buildNettyRemotingClient( - new NettyClientConfig())); + new NettyClientConfig(),null)); public static JdkDynamicRpcClientProxyBuilder withService(Class serviceClazz) { return new JdkDynamicRpcClientProxyBuilder<>(serviceClazz); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index 4aea4d6dfe9c..001b5ed7afcc 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -17,10 +17,13 @@ package org.apache.dolphinscheduler.extract.base.client; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.IRpcResponse; import org.apache.dolphinscheduler.extract.base.SyncRequestDto; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.RemotingException; import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; @@ -33,6 +36,7 @@ import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; +import java.io.File; import java.net.InetSocketAddress; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -54,6 +58,8 @@ import io.netty.channel.socket.SocketChannel; import io.netty.handler.timeout.IdleStateHandler; +import javax.net.ssl.SSLException; + @Slf4j public class NettyRemotingClient implements AutoCloseable { @@ -67,11 +73,21 @@ public class NettyRemotingClient implements AutoCloseable { private final EventLoopGroup workerGroup; private final NettyClientConfig clientConfig; + private final NettySslConfig nettySslConfig; + private SslContext sslContext = null; private final NettyClientHandler clientHandler; - public NettyRemotingClient(final NettyClientConfig clientConfig) { + public NettyRemotingClient(final NettyClientConfig clientConfig , final NettySslConfig nettySslConfig) { this.clientConfig = clientConfig; + this.nettySslConfig = nettySslConfig; + if(nettySslConfig.isEnabled()){ + try { + sslContext = SslContextBuilder.forClient().trustManager(new File(nettySslConfig.getCertFilePath())).build(); + } catch (SSLException e) { + throw new IllegalArgumentException("Initialize SslContext error, please check the cert-file", e); + } + } ThreadFactory nettyClientThreadFactory = ThreadUtils.newDaemonThreadFactory("NettyClientThread-"); if (Epoll.isAvailable()) { this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), nettyClientThreadFactory); @@ -97,6 +113,9 @@ private void start() { @Override public void initChannel(SocketChannel ch) { + if(nettySslConfig.isEnabled()){ + ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + } ch.pipeline() .addLast("client-idle-handler", new IdleStateHandler( diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java index d14a8aa54efc..c7a8a68c7343 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java @@ -21,12 +21,14 @@ import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; @UtilityClass @Slf4j public class NettyRemotingClientFactory { - public NettyRemotingClient buildNettyRemotingClient(NettyClientConfig nettyClientConfig) { - return new NettyRemotingClient(nettyClientConfig); + public NettyRemotingClient buildNettyRemotingClient(NettyClientConfig nettyClientConfig, + NettySslConfig nettySslConfig) { + return new NettyRemotingClient(nettyClientConfig, nettySslConfig); } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java new file mode 100644 index 000000000000..9158d84b072a --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.config; + +import lombok.Data; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties(prefix = "rpc.ssl") +@Data +public class NettySslConfig { + + public boolean enabled; + + public String certFilePath; + + public String keyFilePath; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java index 9ebf802b1ed2..a0cf2754e945 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java @@ -17,13 +17,17 @@ package org.apache.dolphinscheduler.extract.base.server; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.RemoteException; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; +import java.io.File; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; @@ -42,6 +46,8 @@ import io.netty.channel.socket.SocketChannel; import io.netty.handler.timeout.IdleStateHandler; +import javax.net.ssl.SSLException; + /** * remoting netty server */ @@ -66,7 +72,20 @@ class NettyRemotingServer { private final AtomicBoolean isStarted = new AtomicBoolean(false); - NettyRemotingServer(final NettyServerConfig serverConfig) { + private SslContext sslContext = null; + + private NettySslConfig nettySslConfig; + + public NettyRemotingServer(final NettyServerConfig serverConfig, final NettySslConfig nettySslConfig) { + this.nettySslConfig = nettySslConfig; + if (nettySslConfig.isEnabled()) { + try { + sslContext = SslContextBuilder.forServer(new File(nettySslConfig.getCertFilePath()), + new File(nettySslConfig.getKeyFilePath())).build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } this.serverConfig = serverConfig; this.serverName = serverConfig.getServerName(); this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor( @@ -130,6 +149,10 @@ protected void initChannel(SocketChannel ch) { * @param ch socket channel */ private void initNettyChannel(SocketChannel ch) { + if (nettySslConfig.isEnabled()) { + ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc())); + + } ch.pipeline() .addLast("encoder", new TransporterEncoder()) .addLast("decoder", new TransporterDecoder()) diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java index 70ed0529e803..ec0a819d2b37 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java @@ -20,11 +20,13 @@ import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import lombok.experimental.UtilityClass; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; @UtilityClass class NettyRemotingServerFactory { - NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) { - return new NettyRemotingServer(nettyServerConfig); + public NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig, + NettySslConfig nettySslConfig) { + return new NettyRemotingServer(nettyServerConfig, nettySslConfig); } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java index 213868ba46e4..2d57bbb41e7a 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java @@ -24,6 +24,7 @@ import java.lang.reflect.Method; import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; /** * The RpcServer based on Netty. The server will register the method invoker and provide the service to the client. @@ -39,8 +40,8 @@ public class RpcServer implements ServerMethodInvokerRegistry, AutoCloseable { private final NettyRemotingServer nettyRemotingServer; - public RpcServer(NettyServerConfig nettyServerConfig) { - this.nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(nettyServerConfig); + public RpcServer(NettyServerConfig nettyServerConfig, NettySslConfig nettySslConfig) { + this.nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(nettyServerConfig,nettySslConfig); } public void start() { diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java index de4943990cef..aa08d3172fee 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.BeanPostProcessor; import org.springframework.lang.Nullable; @@ -31,8 +32,8 @@ @Slf4j public class SpringServerMethodInvokerDiscovery extends RpcServer implements BeanPostProcessor { - public SpringServerMethodInvokerDiscovery(NettyServerConfig nettyServerConfig) { - super(nettyServerConfig); + public SpringServerMethodInvokerDiscovery(NettyServerConfig nettyServerConfig, NettySslConfig nettySslConfig) { + super(nettyServerConfig,nettySslConfig); } @Nullable diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java index 72a3889b1366..1b80ca22cf67 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java @@ -48,7 +48,7 @@ public void setUp() { .listenPort(listenPort) .build(); serverAddress = "localhost:" + listenPort; - springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig,null); springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new IServiceImpl()); springServerMethodInvokerDiscovery.start(); } diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 1f3c706b8516..2c5fe8ef6e37 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -314,6 +314,10 @@ org.springframework.cloud spring-cloud-starter-kubernetes-client-config + + mysql + mysql-connector-java + diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 4cffd3e37224..953efba7c6fc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -60,7 +61,8 @@ ServiceConfiguration.class, CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class MasterServer implements IStoppable { @@ -94,6 +96,9 @@ public class MasterServer implements IStoppable { @Autowired private SystemEventBusFireWorker systemEventBusFireWorker; + @Autowired + NettySslConfig nettySslConfig; + public static void main(String[] args) { MasterServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java index ab89b021d618..03d5a74b2e3c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.master.rpc; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.server.master.config.MasterConfig; @@ -29,9 +30,9 @@ @Slf4j public class MasterRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { - public MasterRpcServer(MasterConfig masterConfig) { + public MasterRpcServer(MasterConfig masterConfig, NettySslConfig nettySslConfig) { super(NettyServerConfig.builder().serverName("MasterRpcServer").listenPort(masterConfig.getListenPort()) - .build()); + .build(), nettySslConfig); } } diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index 964fca836ce6..6fa5d7f4c1e8 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -16,17 +16,17 @@ # spring: profiles: - active: postgresql + active: mysql banner: charset: UTF-8 jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler - username: root - password: root + driver-class-name: com.mysql.jdbc.Driver + url: jdbc:mysql://10.244.11.111:5555/dolphinscheduler322 + username: dolphin + password: Dolphin!@#123 hikari: connection-test-query: select 1 pool-name: DolphinScheduler @@ -148,16 +148,23 @@ metrics: # Override by profile +rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + --- spring: config: activate: on-profile: mysql datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler - username: root - password: root + driver-class-name: com.mysql.jdbc.Driver + url: jdbc:mysql://10.244.11.111:5555/dolphinscheduler322 + username: dolphin + password: Dolphin!@#123 quartz: properties: org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate + diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index 5ce1423d0d5e..6dcc38588e76 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -74,6 +74,7 @@ + diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java index 1e5a77edb332..68e851fe3468 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.rpc; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.junit.jupiter.api.Assertions; @@ -24,7 +25,7 @@ class MasterRpcServerTest { - private final MasterRpcServer masterRpcServer = new MasterRpcServer(new MasterConfig()); + private final MasterRpcServer masterRpcServer = new MasterRpcServer(new MasterConfig(),new NettySslConfig()); @Test void testStart() { diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java index e9c14face6ce..0d3e8bc03421 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.extract.base.client.Clients; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.microbench.base.AbstractBaseBenchmark; @@ -53,7 +54,7 @@ public class RpcBenchMarkTest extends AbstractBaseBenchmark { public void before() { NettyServerConfig nettyServerConfig = NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build(); - springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig,new NettySslConfig()); springServerMethodInvokerDiscovery.postProcessAfterInitialization(new IServiceImpl(), "iServiceImpl"); springServerMethodInvokerDiscovery.start(); iService = Clients diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 4618c5ad59b6..45ac532428de 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -55,7 +56,8 @@ @Slf4j @Import({CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class WorkerServer implements IStoppable { @@ -71,6 +73,8 @@ public class WorkerServer implements IStoppable { @Autowired private MetricsProvider metricsProvider; + @Autowired + NettySslConfig nettySslConfig; /** * worker server startup, not use web service * @@ -85,6 +89,7 @@ public static void main(String[] args) { @PostConstruct public void run() { +// SingletonJdkDynamicRpcClientProxyFactory.loadInstance(nettySslConfig); ServerLifeCycleManager.toRunning(); this.workerRpcServer.start(); TaskPluginManager.loadTaskPlugin(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java index b9f3855cf952..85f1f8a269b1 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.worker.rpc; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -31,9 +32,9 @@ @Service public class WorkerRpcServer extends SpringServerMethodInvokerDiscovery implements Closeable { - public WorkerRpcServer(WorkerConfig workerConfig) { + public WorkerRpcServer(WorkerConfig workerConfig, NettySslConfig nettySslConfig) { super(NettyServerConfig.builder().serverName("WorkerRpcServer").listenPort(workerConfig.getListenPort()) - .build()); + .build(),nettySslConfig); } } diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index 5cac4c29e5ae..488f89404858 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -92,3 +92,9 @@ management: metrics: enabled: true + +rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java index d27eaeeadfab..8f224da36b68 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.rpc; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.junit.jupiter.api.Assertions; @@ -24,7 +25,7 @@ class WorkerRpcServerTest { - private final WorkerRpcServer workerRpcServer = new WorkerRpcServer(new WorkerConfig()); + private final WorkerRpcServer workerRpcServer = new WorkerRpcServer(new WorkerConfig(),new NettySslConfig()); @Test void testStart() { From 35f5e2b384bbe283c94072a0cac94d4f50f93c38 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 26 Sep 2024 15:49:18 +0800 Subject: [PATCH 2/5] remove unused code --- dolphinscheduler-bom/pom.xml | 4 ++-- dolphinscheduler-master/pom.xml | 4 ---- .../src/main/resources/application.yaml | 20 +++++++++---------- .../src/main/resources/logback-spring.xml | 1 - 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index baccfb4d8e93..2b14225d8e53 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -392,7 +392,7 @@ mysql mysql-connector-java ${mysql-connector.version} - + test @@ -946,7 +946,7 @@ org.testcontainers mysql ${testcontainer.version} - + test diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 2c5fe8ef6e37..1f3c706b8516 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -314,10 +314,6 @@ org.springframework.cloud spring-cloud-starter-kubernetes-client-config - - mysql - mysql-connector-java - diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index 6fa5d7f4c1e8..ff8424910448 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -16,17 +16,17 @@ # spring: profiles: - active: mysql + active: postgresql banner: charset: UTF-8 jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://10.244.11.111:5555/dolphinscheduler322 - username: dolphin - password: Dolphin!@#123 + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root hikari: connection-test-query: select 1 pool-name: DolphinScheduler @@ -153,18 +153,16 @@ rpc: enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem - --- spring: config: activate: on-profile: mysql datasource: - driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://10.244.11.111:5555/dolphinscheduler322 - username: dolphin - password: Dolphin!@#123 + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root quartz: properties: org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate - diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index 6dcc38588e76..5ce1423d0d5e 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -74,7 +74,6 @@ - From c09e9f0c009c1995adbe6b21a983b0ed2f693849 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 26 Sep 2024 15:51:16 +0800 Subject: [PATCH 3/5] mvn spotless:apply --- .../alert/rpc/AlertRpcServer.java | 3 ++- .../alert/rpc/AlertRpcServerTest.java | 4 ++-- .../api/service/LoggerServiceTest.java | 3 ++- .../extract/base/client/Clients.java | 2 +- .../base/client/NettyRemotingClient.java | 17 +++++++++-------- .../base/client/NettyRemotingClientFactory.java | 2 +- .../base/server/NettyRemotingServer.java | 8 ++++---- .../base/server/NettyRemotingServerFactory.java | 2 +- .../extract/base/server/RpcServer.java | 5 +++-- .../SpringServerMethodInvokerDiscovery.java | 4 ++-- .../extract/base/client/ClientsTest.java | 2 +- .../server/master/rpc/MasterRpcServerTest.java | 2 +- .../microbench/rpc/RpcBenchMarkTest.java | 3 ++- .../server/worker/WorkerServer.java | 2 +- .../server/worker/rpc/WorkerRpcServer.java | 2 +- .../server/worker/rpc/WorkerRpcServerTest.java | 2 +- 16 files changed, 34 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java index aeaf100deb99..0cdb9d578b41 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java @@ -31,7 +31,8 @@ public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { public AlertRpcServer(AlertConfig alertConfig, NettySslConfig nettySslConfig) { - super(NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build(),nettySslConfig); + super(NettyServerConfig.builder().serverName("AlertRpcServer").listenPort(alertConfig.getPort()).build(), + nettySslConfig); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java index cdd905cbbf11..0c1cd0a3c9d4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServerTest.java @@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.alert.rpc; import org.apache.dolphinscheduler.alert.config.AlertConfig; - import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; + import org.junit.jupiter.api.Test; class AlertRpcServerTest { - private final AlertRpcServer alertRpcServer = new AlertRpcServer(new AlertConfig(),new NettySslConfig()); + private final AlertRpcServer alertRpcServer = new AlertRpcServer(new AlertConfig(), new NettySslConfig()); @Test void testStart() { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index c20f25c54935..282f27f33f1d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -104,7 +104,8 @@ public void setUp() { } springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery( - NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build(),new NettySslConfig()); + NettyServerConfig.builder().serverName("TestLogServer").listenPort(nettyServerPort).build(), + new NettySslConfig()); springServerMethodInvokerDiscovery.start(); springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new ILogService() { diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java index 50ea4b589735..077115c0c5d9 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/Clients.java @@ -36,7 +36,7 @@ public class Clients { private static final JdkDynamicRpcClientProxyFactory jdkDynamicRpcClientProxyFactory = new JdkDynamicRpcClientProxyFactory( NettyRemotingClientFactory.buildNettyRemotingClient( - new NettyClientConfig(),null)); + new NettyClientConfig(), null)); public static JdkDynamicRpcClientProxyBuilder withService(Class serviceClazz) { return new JdkDynamicRpcClientProxyBuilder<>(serviceClazz); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index 001b5ed7afcc..2c3f0dca30c2 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.extract.base.client; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.IRpcResponse; import org.apache.dolphinscheduler.extract.base.SyncRequestDto; @@ -45,6 +43,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; +import javax.net.ssl.SSLException; + import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -56,10 +56,10 @@ import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.timeout.IdleStateHandler; -import javax.net.ssl.SSLException; - @Slf4j public class NettyRemotingClient implements AutoCloseable { @@ -78,12 +78,13 @@ public class NettyRemotingClient implements AutoCloseable { private final NettyClientHandler clientHandler; - public NettyRemotingClient(final NettyClientConfig clientConfig , final NettySslConfig nettySslConfig) { + public NettyRemotingClient(final NettyClientConfig clientConfig, final NettySslConfig nettySslConfig) { this.clientConfig = clientConfig; this.nettySslConfig = nettySslConfig; - if(nettySslConfig.isEnabled()){ + if (nettySslConfig.isEnabled()) { try { - sslContext = SslContextBuilder.forClient().trustManager(new File(nettySslConfig.getCertFilePath())).build(); + sslContext = + SslContextBuilder.forClient().trustManager(new File(nettySslConfig.getCertFilePath())).build(); } catch (SSLException e) { throw new IllegalArgumentException("Initialize SslContext error, please check the cert-file", e); } @@ -113,7 +114,7 @@ private void start() { @Override public void initChannel(SocketChannel ch) { - if(nettySslConfig.isEnabled()){ + if (nettySslConfig.isEnabled()) { ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); } ch.pipeline() diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java index c7a8a68c7343..f458ca354409 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClientFactory.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.extract.base.client; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; @UtilityClass @Slf4j diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java index a0cf2754e945..eef9a7a46f5b 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.extract.base.server; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; @@ -33,6 +31,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import javax.net.ssl.SSLException; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.ServerBootstrap; @@ -44,10 +44,10 @@ import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.timeout.IdleStateHandler; -import javax.net.ssl.SSLException; - /** * remoting netty server */ diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java index ec0a819d2b37..2202e96e7853 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServerFactory.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.extract.base.server; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import lombok.experimental.UtilityClass; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; @UtilityClass class NettyRemotingServerFactory { diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java index 2d57bbb41e7a..ad4baedb32d9 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/RpcServer.java @@ -20,11 +20,11 @@ import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.RpcService; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import java.lang.reflect.Method; import lombok.extern.slf4j.Slf4j; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; /** * The RpcServer based on Netty. The server will register the method invoker and provide the service to the client. @@ -41,7 +41,8 @@ public class RpcServer implements ServerMethodInvokerRegistry, AutoCloseable { private final NettyRemotingServer nettyRemotingServer; public RpcServer(NettyServerConfig nettyServerConfig, NettySslConfig nettySslConfig) { - this.nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(nettyServerConfig,nettySslConfig); + this.nettyRemotingServer = + NettyRemotingServerFactory.buildNettyRemotingServer(nettyServerConfig, nettySslConfig); } public void start() { diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java index aa08d3172fee..d46a291c194b 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.extract.base.server; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import lombok.extern.slf4j.Slf4j; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.springframework.beans.BeansException; import org.springframework.beans.factory.config.BeanPostProcessor; import org.springframework.lang.Nullable; @@ -33,7 +33,7 @@ public class SpringServerMethodInvokerDiscovery extends RpcServer implements BeanPostProcessor { public SpringServerMethodInvokerDiscovery(NettyServerConfig nettyServerConfig, NettySslConfig nettySslConfig) { - super(nettyServerConfig,nettySslConfig); + super(nettyServerConfig, nettySslConfig); } @Nullable diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java index 1b80ca22cf67..e86c4e58dc78 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java @@ -48,7 +48,7 @@ public void setUp() { .listenPort(listenPort) .build(); serverAddress = "localhost:" + listenPort; - springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig,null); + springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig, null); springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new IServiceImpl()); springServerMethodInvokerDiscovery.start(); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java index 68e851fe3468..2d93e61fdec0 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServerTest.java @@ -25,7 +25,7 @@ class MasterRpcServerTest { - private final MasterRpcServer masterRpcServer = new MasterRpcServer(new MasterConfig(),new NettySslConfig()); + private final MasterRpcServer masterRpcServer = new MasterRpcServer(new MasterConfig(), new NettySslConfig()); @Test void testStart() { diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java index 0d3e8bc03421..8787f9735d44 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/rpc/RpcBenchMarkTest.java @@ -54,7 +54,8 @@ public class RpcBenchMarkTest extends AbstractBaseBenchmark { public void before() { NettyServerConfig nettyServerConfig = NettyServerConfig.builder().serverName("NettyRemotingServer").listenPort(12345).build(); - springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig,new NettySslConfig()); + springServerMethodInvokerDiscovery = + new SpringServerMethodInvokerDiscovery(nettyServerConfig, new NettySslConfig()); springServerMethodInvokerDiscovery.postProcessAfterInitialization(new IServiceImpl(), "iServiceImpl"); springServerMethodInvokerDiscovery.start(); iService = Clients diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 45ac532428de..14167cb4d06e 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -89,7 +89,7 @@ public static void main(String[] args) { @PostConstruct public void run() { -// SingletonJdkDynamicRpcClientProxyFactory.loadInstance(nettySslConfig); + // SingletonJdkDynamicRpcClientProxyFactory.loadInstance(nettySslConfig); ServerLifeCycleManager.toRunning(); this.workerRpcServer.start(); TaskPluginManager.loadTaskPlugin(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java index 85f1f8a269b1..536cfbe98e23 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java @@ -34,7 +34,7 @@ public class WorkerRpcServer extends SpringServerMethodInvokerDiscovery implemen public WorkerRpcServer(WorkerConfig workerConfig, NettySslConfig nettySslConfig) { super(NettyServerConfig.builder().serverName("WorkerRpcServer").listenPort(workerConfig.getListenPort()) - .build(),nettySslConfig); + .build(), nettySslConfig); } } diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java index 8f224da36b68..b9716a706858 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServerTest.java @@ -25,7 +25,7 @@ class WorkerRpcServerTest { - private final WorkerRpcServer workerRpcServer = new WorkerRpcServer(new WorkerConfig(),new NettySslConfig()); + private final WorkerRpcServer workerRpcServer = new WorkerRpcServer(new WorkerConfig(), new NettySslConfig()); @Test void testStart() { From 319f701eb5531f8e6eaf5db9bcd3455ee1965cde Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 26 Sep 2024 15:52:01 +0800 Subject: [PATCH 4/5] remove test code --- .../org/apache/dolphinscheduler/server/worker/WorkerServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 14167cb4d06e..f833967de630 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -89,7 +89,6 @@ public static void main(String[] args) { @PostConstruct public void run() { - // SingletonJdkDynamicRpcClientProxyFactory.loadInstance(nettySslConfig); ServerLifeCycleManager.toRunning(); this.workerRpcServer.start(); TaskPluginManager.loadTaskPlugin(); From da9ae3ef390672801facaf4303451a39c2f693d9 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 26 Sep 2024 16:12:43 +0800 Subject: [PATCH 5/5] fix ut --- .../extract/base/client/ClientsTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java index e86c4e58dc78..bd1d63a01825 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/test/java/org/apache/dolphinscheduler/extract/base/client/ClientsTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.extract.base.RpcMethod; import org.apache.dolphinscheduler.extract.base.RpcService; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.MethodInvocationException; import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; @@ -48,7 +49,8 @@ public void setUp() { .listenPort(listenPort) .build(); serverAddress = "localhost:" + listenPort; - springServerMethodInvokerDiscovery = new SpringServerMethodInvokerDiscovery(nettyServerConfig, null); + springServerMethodInvokerDiscovery = + new SpringServerMethodInvokerDiscovery(nettyServerConfig, getNettySslConfig()); springServerMethodInvokerDiscovery.registerServerMethodInvokerProvider(new IServiceImpl()); springServerMethodInvokerDiscovery.start(); } @@ -111,5 +113,9 @@ public void voidMethod() { System.out.println("void method"); } } - + private NettySslConfig getNettySslConfig() { + NettySslConfig info = new NettySslConfig(); + info.setEnabled(false); + return info; + } }