diff --git a/.asf.yaml b/.asf.yaml index 556a0c2658b0..d65f2758badf 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -47,6 +47,7 @@ github: - Docs - Frontend Build - "E2E-K8S-Result" + - "API-Test-Result" - "Mergeable: milestone-label-check" - "Title Validator" required_pull_request_reviews: diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml index 90a2e9aa1776..e9923fd59c7c 100644 --- a/.github/workflows/api-test.yml +++ b/.github/workflows/api-test.yml @@ -51,6 +51,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - uses: actions/checkout@v4 with: submodules: true @@ -67,8 +80,8 @@ jobs: - name: Build Image run: | ./mvnw -B clean install \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ + -Dmaven.test.skip=true \ + -Dmaven.javadoc.skip=true \ -Dspotless.skip=true \ -Pdocker,release -Ddocker.tag=ci - name: Export Docker Images diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index e7b282a29c68..268c313fe7ca 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -56,6 +56,19 @@ jobs: java: [ '8', '11' ] timeout-minutes: 30 steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - uses: actions/checkout@v4 with: submodules: true @@ -78,10 +91,7 @@ jobs: ./mvnw -B clean install \ -Prelease \ -Dmaven.test.skip=true \ - -Dspotless.skip=true \ - -Dhttp.keepAlive=false \ - -Dmaven.wagon.http.pool=false \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + -Dspotless.skip=true - name: Check dependency license run: tools/dependencies/check-LICENSE.sh - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile index 902003625b87..b0b51a43a967 100644 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile +++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile @@ -22,22 +22,24 @@ RUN apt update ; \ COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ +RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin +ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting dolphinscheduler_env.sh COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Download mysql jar -ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" -ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" -RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER +ENV MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +ENV MYSQL_DRIVER="mysql-connector-java-8.0.16.jar" +RUN wget -O $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $MYSQL_URL +RUN cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER #Deploy COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh deleted file mode 100644 index cd660febf88b..000000000000 --- a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile index e2eda217c76d..0c1e8a7c006c 100644 --- a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile +++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile @@ -22,22 +22,24 @@ RUN apt update ; \ COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ +RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin +ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting dolphinscheduler_env.sh COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh #Download mysql jar -ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" -ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" -RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ -cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER +ENV MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +ENV MYSQL_DRIVER="mysql-connector-java-8.0.16.jar" +RUN wget -O $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $MYSQL_URL +RUN cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ + cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER #Deploy COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile index 4fc915ab0d7c..cc8e0e21d222 100644 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile +++ b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile @@ -22,10 +22,11 @@ RUN apt update ; \ COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ +RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin +ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting dolphinscheduler_env.sh COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh deleted file mode 100644 index cd660febf88b..000000000000 --- a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile index 5d81ce2fb48f..b84c46df88d1 100644 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile +++ b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile @@ -22,10 +22,11 @@ RUN apt update ; \ COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~ +RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin -ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin +ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin #Setting dolphinscheduler_env.sh COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh deleted file mode 100644 index e43813479bb1..000000000000 --- a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh +++ /dev/null @@ -1,61 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --------------------------------------------------------- -# INSTALL MACHINE -# --------------------------------------------------------- -# A comma separated list of machine hostname or IP would be installed DolphinScheduler, -# including master, worker, api, alert. If you want to deploy in pseudo-distributed -# mode, just write a pseudo-distributed hostname -# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" -ips=${ips:-"localhost"} - -# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine -# modify it if you use different ssh port -sshPort=${sshPort:-"22"} - -# A comma separated list of machine hostname or IP would be installed Master server, it -# must be a subset of configuration `ips`. -# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" -masters=${masters:-"localhost"} - -# A comma separated list of machine : or :.All hostname or IP must be a -# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts -# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" -workers=${workers:-"localhost:default"} - -# A comma separated list of machine hostname or IP would be installed Alert server, it -# must be a subset of configuration `ips`. -# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" -alertServer=${alertServer:-"localhost"} - -# A comma separated list of machine hostname or IP would be installed API server, it -# must be a subset of configuration `ips`. -# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" -apiServers=${apiServers:-"localhost"} - -# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. -# Do not set this configuration same as the current path (pwd) -installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} - -# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` -# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs -# to be created by this user -deployUser=${deployUser:-"dolphinscheduler"} - -# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. -zkRoot=${zkRoot:-"/dolphinscheduler"} diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 653ae36edf40..e5ad17ff86b0 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -49,26 +49,31 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - uses: actions/checkout@v4 with: submodules: true - name: Build Image run: | ./mvnw -B clean package \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ + -Dmaven.test.skip=true \ + -Dmaven.javadoc.skip=true \ -Dspotless.skip=true \ - -Dmaven.checkstyle.skip \ - -Dmaven.deploy.skip \ - -Ddocker.push.skip=true \ - -Pdocker,release -Ddocker.tag=ci \ - -pl org.apache.dolphinscheduler:dolphinscheduler-alert-server \ - -pl dolphinscheduler-tools \ - -pl dolphinscheduler-api \ - -pl dolphinscheduler-master \ - -pl dolphinscheduler-worker -am + -Pdocker,release -Ddocker.tag=ci - name: Create k8s Kind Cluster run: | # install kubectl diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a24cf820f8c4..8dc83efdc9f8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -53,6 +53,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - uses: actions/checkout@v4 with: submodules: true @@ -69,11 +82,10 @@ jobs: - name: Build Image run: | ./mvnw -B clean install \ - -Dmaven.test.skip \ - -Dmaven.javadoc.skip \ + -Dmaven.test.skip=true \ + -Dmaven.javadoc.skip=true \ -Dspotless.skip=true \ - -Pdocker,release -Ddocker.tag=ci \ - -pl dolphinscheduler-standalone-server -am + -Pdocker,release -Ddocker.tag=ci - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index 19eaab9f2561..2d04204d7a74 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -87,7 +87,6 @@ jobs: -Dmaven.test.skip \ -Dmaven.javadoc.skip \ -Dspotless.skip=true \ - -Dmaven.deploy.skip \ -Ddocker.tag=${{ env.DOCKER_TAG }} \ -Ddocker.hub=${{ env.HUB }} \ -Pdocker,release diff --git a/docs/docs/en/contribute/development-environment-setup.md b/docs/docs/en/contribute/development-environment-setup.md index 53e92ba7b78d..5da55fced292 100644 --- a/docs/docs/en/contribute/development-environment-setup.md +++ b/docs/docs/en/contribute/development-environment-setup.md @@ -80,7 +80,6 @@ cd dolphinscheduler -Dmaven.test.skip \ -Dmaven.javadoc.skip \ -Dspotless.skip = true \ - -Dmaven.deploy.skip \ -Ddocker.tag= \ -Ddocker.hub= \ -Pdocker,release diff --git a/docs/docs/zh/contribute/development-environment-setup.md b/docs/docs/zh/contribute/development-environment-setup.md index 8e7fe502c3c9..d6b56de30d43 100644 --- a/docs/docs/zh/contribute/development-environment-setup.md +++ b/docs/docs/zh/contribute/development-environment-setup.md @@ -77,7 +77,6 @@ cd dolphinscheduler -Dmaven.test.skip \ -Dmaven.javadoc.skip \ -Dspotless.skip = true \ - -Dmaven.deploy.skip \ -Ddocker.tag= \ -Ddocker.hub= \ -Pdocker,release diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/pom.xml index e7dd834a5f88..21a160a95ab7 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-aliyunVoice/pom.xml @@ -27,14 +27,43 @@ dolphinscheduler-alert-aliyunVoice jar + + alert.aliyun.voice + + org.apache.dolphinscheduler dolphinscheduler-alert-api + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + com.aliyun dyvmsapi20170525 + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/pom.xml index 4ce63942bdbc..d6680c1d78e5 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-api/pom.xml @@ -31,11 +31,12 @@ org.apache.dolphinscheduler dolphinscheduler-spi - + - org.springframework - spring-context + org.apache.dolphinscheduler + dolphinscheduler-common + provided diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml index db64f7e4f2eb..f4206b3eb3d7 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml @@ -26,14 +26,38 @@ dolphinscheduler-alert-dingtalk jar + + alert.dingtalk + + org.apache.dolphinscheduler dolphinscheduler-alert-api + - org.apache.httpcomponents - httpclient + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml index e088d334580b..123da7474715 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml @@ -26,11 +26,23 @@ dolphinscheduler-alert-email jar + + alert.email + + org.apache.dolphinscheduler dolphinscheduler-alert-api + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + org.apache.poi poi @@ -45,4 +57,21 @@ commons-email + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml index 98ab229d1a01..b10ecbf111eb 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml @@ -26,14 +26,38 @@ dolphinscheduler-alert-feishu jar + + alert.feishu + + org.apache.dolphinscheduler dolphinscheduler-alert-api + - org.apache.httpcomponents - httpclient + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml index 274ec7f1d918..6f4fade79f00 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml @@ -26,19 +26,20 @@ dolphinscheduler-alert-http jar + + alert.http + + org.apache.dolphinscheduler dolphinscheduler-alert-api - - org.apache.httpcomponents - httpclient - - com.fasterxml.jackson.core - jackson-databind + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} provided @@ -54,4 +55,21 @@ + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml index b356e7870395..9a17ad298fed 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml @@ -26,19 +26,38 @@ dolphinscheduler-alert-pagerduty jar + + alert.pagerduty + + org.apache.dolphinscheduler dolphinscheduler-alert-api - - com.google.guava - guava - - org.apache.httpcomponents - httpclient + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/pom.xml index 8182b2fa4988..568fcddcaf13 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-prometheus/pom.xml @@ -26,14 +26,38 @@ dolphinscheduler-alert-prometheus jar + + alert.prometheus + + org.apache.dolphinscheduler dolphinscheduler-alert-api + - org.apache.httpcomponents - httpclient + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml index 9a783cb2d575..c3487b3ddff6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml @@ -26,10 +26,38 @@ dolphinscheduler-alert-script jar + + alert.script + + org.apache.dolphinscheduler dolphinscheduler-alert-api + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml index 39594d351d67..2cd6c1fd9634 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml @@ -26,19 +26,38 @@ dolphinscheduler-alert-slack jar + + alert.slack + + org.apache.dolphinscheduler dolphinscheduler-alert-api - - org.apache.httpcomponents - httpclient - - com.google.guava - guava + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml index cd96676e68d7..0e174abddfae 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml @@ -26,14 +26,39 @@ dolphinscheduler-alert-telegram jar + + alert.telegram + + org.apache.dolphinscheduler dolphinscheduler-alert-api + provided + - org.apache.httpcomponents - httpclient + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml index 99bc262c7033..add5b0194fd6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml @@ -26,19 +26,38 @@ dolphinscheduler-alert-webexteams jar + + alert.webexteams + + org.apache.dolphinscheduler dolphinscheduler-alert-api - - com.google.guava - guava - - org.apache.httpcomponents - httpclient + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml index 733ac16cf0c7..b6c2df63ed9c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml @@ -26,20 +26,38 @@ dolphinscheduler-alert-wechat jar + + alert.wechat + + org.apache.dolphinscheduler dolphinscheduler-alert-api - - org.apache.httpcomponents - httpclient - - com.fasterxml.jackson.core - jackson-databind + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 844a5983df8f..246e69193949 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -33,6 +33,7 @@ org.apache.dolphinscheduler dolphinscheduler-meter + org.apache.dolphinscheduler dolphinscheduler-alert-all @@ -53,6 +54,11 @@ dolphinscheduler-extract-alert + + org.apache.dolphinscheduler + dolphinscheduler-common + + com.google.guava guava diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index 24c8fb2f11fe..df39cd51d363 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -57,13 +57,14 @@ conf - ${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources + ${basedir}/../../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources **/*.yaml conf + libs diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh index bfd29abcc1c4..56a493d9b7ed 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh @@ -18,11 +18,12 @@ set -eo pipefail BIN_DIR=$(dirname $(readlink -f "$0")) -DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd) +DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/../..;pwd) +ALERT_HOME=$(cd ${BIN_DIR}/..;pwd) -source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" +source "$ALERT_HOME/conf/dolphinscheduler_env.sh" -JVM_ARGS_ENV_FILE=${BIN_DIR}/bin/jvm_args_env.sh +JVM_ARGS_ENV_FILE=${ALERT_HOME}/bin/jvm_args_env.sh JVM_ARGS="-server" if [ -f $JVM_ARGS_ENV_FILE ]; then @@ -37,12 +38,30 @@ fi JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then - JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" + JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport -DDOCKER=true" fi echo "JAVA_HOME=${JAVA_HOME}" echo "JAVA_OPTS=${JAVA_OPTS}" +MODULES_PATH=( +alert-server +) + +CP="" +for module in ${MODULES_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/$module/libs/*" +done + +PLUGINS_PATH=( +alert-plugins +) + +for plugin in ${PLUGINS_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" +done + + $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ + -cp "$ALERT_HOME/conf""$CP" \ org.apache.dolphinscheduler.alert.AlertServer diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml index 054e1bd48244..67e79d8653da 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml @@ -28,7 +28,7 @@ services: - api-test volumes: - ./download-mysql.sh:/tmp/download-mysql.sh - entrypoint: [ 'bash', '-c', '/bin/bash /tmp/download-mysql.sh && /opt/dolphinscheduler/bin/start.sh && tail -f /dev/null' ] + entrypoint: [ 'bash', '-c', '/bin/bash /tmp/download-mysql.sh && /bin/bash /opt/dolphinscheduler/standalone-server/bin/start.sh && tail -f /dev/null' ] healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s @@ -52,4 +52,4 @@ services: retries: 120 networks: - api-test: \ No newline at end of file + api-test: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh index ef0ea7309c00..ff848739cd55 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh @@ -17,7 +17,7 @@ set -ex -DS_HOME=/opt/dolphinscheduler/libs/standalone-server +DS_HOME=/opt/dolphinscheduler/libs MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" MYSQL_DRIVER="mysql-connector-java-8.0.16.jar" diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml index 8dca2fbf03ae..dbe25bcbadf6 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml @@ -32,7 +32,7 @@ services: timeout: 60s retries: 120 volumes: - - ./common.properties:/opt/dolphinscheduler/conf/common.properties + - ./common.properties:/opt/dolphinscheduler/standalone-server/conf/common.properties depends_on: s3: condition: service_healthy diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 40d556a17ed3..7afcf6b89494 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -282,18 +282,4 @@ - - - - docker - - - - org.codehaus.mojo - exec-maven-plugin - - - - - diff --git a/dolphinscheduler-api/src/main/bin/start.sh b/dolphinscheduler-api/src/main/bin/start.sh index 7782f9c47bb7..eb60455c5350 100644 --- a/dolphinscheduler-api/src/main/bin/start.sh +++ b/dolphinscheduler-api/src/main/bin/start.sh @@ -18,11 +18,12 @@ set -eo pipefail BIN_DIR=$(dirname $(readlink -f "$0")) -DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd) +DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/../..;pwd) +API_HOME=$(cd ${BIN_DIR}/..;pwd) -source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" +source "$API_HOME/conf/dolphinscheduler_env.sh" -JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh +JVM_ARGS_ENV_FILE=${API_HOME}/bin/jvm_args_env.sh JVM_ARGS="-server" if [ -f $JVM_ARGS_ENV_FILE ]; then @@ -43,6 +44,25 @@ fi echo "JAVA_HOME=${JAVA_HOME}" echo "JAVA_OPTS=${JAVA_OPTS}" +MODULES_PATH=( +api-server +) + +CP="" +for module in ${MODULES_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/$module/libs/*" +done + +PLUGINS_PATH=( +datasource-plugins +storage-plugins +task-plugins +) + +for plugin in ${PLUGINS_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" +done + $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ + -cp "$API_HOME/conf""$CP" \ org.apache.dolphinscheduler.api.ApiApplicationServer diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index b7efb4815cf1..2b14225d8e53 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -90,7 +90,6 @@ 3.15.1 2.10.13 4.12.0 - 2.7.0 2021.0.3 2.9.1 4.2.11 @@ -732,12 +731,6 @@ ${joda-time.version} - - com.jayway.jsonpath - json-path - ${json-path.version} - - org.springframework.cloud spring-cloud-dependencies diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 6c83c580ae25..74a17da892df 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -45,6 +45,7 @@ org.apache.dolphinscheduler dolphinscheduler-aws-authentication + commons-io commons-io @@ -74,14 +75,22 @@ com.fasterxml.jackson.core jackson-databind + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + org.apache.commons commons-collections4 + commons-beanutils commons-beanutils + org.apache.commons commons-lang3 @@ -92,11 +101,6 @@ aliyun-sdk-oss - - com.amazonaws - aws-java-sdk-s3 - - com.huaweicloud esdk-obs-java-bundle @@ -105,6 +109,12 @@ com.azure azure-storage-blob + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + @@ -186,10 +196,5 @@ - - - org.javadelight - delight-nashorn-sandbox - diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java index b5ab837a8415..ff44bc866d63 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.common.exception; -/** - * exception for store - */ public class StorageOperateNoConfiguredException extends RuntimeException { public StorageOperateNoConfiguredException() { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/pom.xml index 80226fe3670f..e6ba7801297e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.aliyun.serverlessspark + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -54,4 +60,20 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml index a80509f46ee8..6955a03eb884 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml @@ -35,42 +35,7 @@ org.apache.dolphinscheduler dolphinscheduler-common - - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - - - - commons-io - commons-io - - - - org.slf4j - slf4j-api - provided - - - - com.google.guava - guava - - - - commons-codec - commons-codec - - - - org.apache.commons - commons-collections4 - - - - com.github.oshi - oshi-core + ${project.version} provided @@ -79,11 +44,6 @@ spring-jdbc - - org.apache.hbase.thirdparty - hbase-noop-htrace - - org.apache.hadoop hadoop-client @@ -145,6 +105,5 @@ com.alibaba druid - diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/BasePooledDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/BasePooledDataSourceClient.java index f5078df121e4..f7a27e9acca4 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/BasePooledDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/BasePooledDataSourceClient.java @@ -19,8 +19,8 @@ import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/constants/DataSourceConstants.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/constants/DataSourceConstants.java new file mode 100644 index 000000000000..18ae55d7e9c4 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/constants/DataSourceConstants.java @@ -0,0 +1,477 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.api.constants; + +import org.apache.dolphinscheduler.common.constants.DateConstants; + +import java.time.Duration; +import java.util.Set; +import java.util.regex.Pattern; + +import lombok.experimental.UtilityClass; + +import com.google.common.collect.Sets; + +@UtilityClass +public class DataSourceConstants { + + public static final String ORG_POSTGRESQL_DRIVER = "org.postgresql.Driver"; + public static final String COM_MYSQL_CJ_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; + public static final String COM_MYSQL_JDBC_DRIVER = "com.mysql.jdbc.Driver"; + public static final String ORG_APACHE_HIVE_JDBC_HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; + public static final String COM_CLICKHOUSE_JDBC_DRIVER = "com.clickhouse.jdbc.ClickHouseDriver"; + public static final String COM_DATABEND_JDBC_DRIVER = "com.databend.jdbc.DatabendDriver"; + public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.OracleDriver"; + public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; + public static final String COM_REDSHIFT_JDBC_DRIVER = "com.amazon.redshift.jdbc42.Driver"; + public static final String COM_ATHENA_JDBC_DRIVER = "com.simba.athena.jdbc.Driver"; + public static final String COM_TRINO_JDBC_DRIVER = "io.trino.jdbc.TrinoDriver"; + public static final String COM_DAMENG_JDBC_DRIVER = "dm.jdbc.driver.DmDriver"; + public static final String ORG_APACHE_KYUUBI_JDBC_DRIVER = "org.apache.kyuubi.jdbc.KyuubiHiveDriver"; + public static final String COM_OCEANBASE_JDBC_DRIVER = "com.oceanbase.jdbc.Driver"; + public static final String NET_SNOWFLAKE_JDBC_DRIVER = "net.snowflake.client.jdbc.SnowflakeDriver"; + public static final String COM_VERTICA_JDBC_DRIVER = "com.vertica.jdbc.Driver"; + public static final String COM_HANA_DB_JDBC_DRIVER = "com.sap.db.jdbc.Driver"; + + public static final String JDBC_MYSQL = "jdbc:mysql://"; + public static final String JDBC_MYSQL_LOADBALANCE = "jdbc:mysql:loadbalance://"; + public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; + public static final String JDBC_HIVE_2 = "jdbc:hive2://"; + public static final String JDBC_KYUUBI = "jdbc:kyuubi://"; + public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; + public static final String JDBC_DATABEND = "jdbc:databend://"; + public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; + public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; + public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; + public static final String JDBC_DB2 = "jdbc:db2://"; + public static final String JDBC_PRESTO = "jdbc:presto://"; + public static final String JDBC_REDSHIFT = "jdbc:redshift://"; + public static final String JDBC_REDSHIFT_IAM = "jdbc:redshift:iam://"; + public static final String JDBC_ATHENA = "jdbc:awsathena://"; + public static final String JDBC_TRINO = "jdbc:trino://"; + public static final String JDBC_DAMENG = "jdbc:dm://"; + public static final String JDBC_OCEANBASE = "jdbc:oceanbase://"; + public static final String JDBC_SNOWFLAKE = "jdbc:snowflake://"; + public static final String JDBC_VERTICA = "jdbc:vertica://"; + public static final String JDBC_HANA = "jdbc:sap://"; + + public static final String POSTGRESQL_VALIDATION_QUERY = "select version()"; + public static final String MYSQL_VALIDATION_QUERY = "select 1"; + public static final String HIVE_VALIDATION_QUERY = "select 1"; + public static final String CLICKHOUSE_VALIDATION_QUERY = "select 1"; + public static final String DATABEND_VALIDATION_QUERY = "select 1"; + public static final String ORACLE_VALIDATION_QUERY = "select 1 from dual"; + public static final String SQLSERVER_VALIDATION_QUERY = "select 1"; + public static final String DB2_VALIDATION_QUERY = "select 1 from sysibm.sysdummy1"; + public static final String PRESTO_VALIDATION_QUERY = "select 1"; + public static final String REDHIFT_VALIDATION_QUERY = "select 1"; + public static final String ATHENA_VALIDATION_QUERY = "select 1"; + public static final String TRINO_VALIDATION_QUERY = "select 1"; + public static final String DAMENG_VALIDATION_QUERY = "select 1"; + public static final String SNOWFLAKE_VALIDATION_QUERY = "select 1"; + + public static final String KYUUBI_VALIDATION_QUERY = "select 1"; + public static final String VERTICA_VALIDATION_QUERY = "select 1"; + + public static final String HANA_VALIDATION_QUERY = "select 1 from DUMMY"; + + public static final String SPRING_DATASOURCE_MIN_IDLE = "spring.datasource.minIdle"; + + public static final String SPRING_DATASOURCE_MAX_ACTIVE = "spring.datasource.maxActive"; + + public static final String SUPPORT_HIVE_ONE_SESSION = "support.hive.oneSession"; + /** + * QUESTION ? + */ + public static final String QUESTION = "?"; + + /** + * comma , + */ + public static final String COMMA = ","; + + /** + * hyphen + */ + public static final String HYPHEN = "-"; + + /** + * slash / + */ + public static final String SLASH = "/"; + + /** + * COLON : + */ + public static final String COLON = ":"; + + /** + * SPACE " " + */ + public static final String SPACE = " "; + + /** + * SINGLE_SLASH / + */ + public static final String SINGLE_SLASH = "/"; + + /** + * DOUBLE_SLASH // + */ + public static final String DOUBLE_SLASH = "//"; + + /** + * SINGLE_QUOTES "'" + */ + public static final String SINGLE_QUOTES = "'"; + /** + * DOUBLE_QUOTES "\"" + */ + public static final String DOUBLE_QUOTES = "\""; + + /** + * SEMICOLON ; + */ + public static final String SEMICOLON = ";"; + + /** + * EQUAL SIGN + */ + public static final String EQUAL_SIGN = "="; + /** + * AT SIGN + */ + public static final String AT_SIGN = "@"; + /** + * UNDERLINE + */ + public static final String UNDERLINE = "_"; + + /** + * sleep time + */ + public static final int SLEEP_TIME_MILLIS = 1000; + + /** + * exit code failure + */ + public static final int EXIT_CODE_FAILURE = -1; + + /** + * exit code success + */ + public static final int EXIT_CODE_SUCCESS = 0; + /** + * running code + */ + public static final int RUNNING_CODE = 1; + + public static final String SH = "sh"; + + /** + * log flush interval?output when reach the interval + */ + public static final int DEFAULT_LOG_FLUSH_INTERVAL = 1000; + + /** + * pstree, get pud and sub pid + */ + public static final String PSTREE = "pstree"; + + public static final String RWXR_XR_X = "rwxr-xr-x"; + + /** + * date format of yyyyMMddHHmmss + */ + public static final String PARAMETER_FORMAT_TIME = "yyyyMMddHHmmss"; + + /** + * new + * schedule time + */ + public static final String PARAMETER_SHECDULE_TIME = "schedule.time"; + + /** + * system date(yyyyMMddHHmmss) + */ + public static final String PARAMETER_DATETIME = DateConstants.PARAMETER_DATETIME; + + /** + * system date(yyyymmdd) today + */ + public static final String PARAMETER_CURRENT_DATE = DateConstants.PARAMETER_CURRENT_DATE; + + /** + * system date(yyyymmdd) yesterday + */ + public static final String PARAMETER_BUSINESS_DATE = DateConstants.PARAMETER_BUSINESS_DATE; + + /** + * the absolute path of current executing task + */ + public static final String PARAMETER_TASK_EXECUTE_PATH = "system.task.execute.path"; + + /** + * the instance id of current task + */ + public static final String PARAMETER_TASK_INSTANCE_ID = "system.task.instance.id"; + + /** + * the definition code of current task + */ + public static final String PARAMETER_TASK_DEFINITION_CODE = "system.task.definition.code"; + + /** + * the definition name of current task + */ + public static final String PARAMETER_TASK_DEFINITION_NAME = "system.task.definition.name"; + + /** + * the instance id of the workflow to which current task belongs + */ + public static final String PARAMETER_WORKFLOW_INSTANCE_ID = "system.workflow.instance.id"; + + /** + * the definition code of the workflow to which current task belongs + */ + public static final String PARAMETER_WORKFLOW_DEFINITION_CODE = "system.workflow.definition.code"; + + /** + * the definition name of the workflow to which current task belongs + */ + public static final String PARAMETER_WORKFLOW_DEFINITION_NAME = "system.workflow.definition.name"; + + /** + * the code of the project to which current task belongs + */ + public static final String PARAMETER_PROJECT_CODE = "system.project.code"; + + /** + * the name of the project to which current task belongs + */ + public static final String PARAMETER_PROJECT_NAME = "system.project.name"; + /** + * month_begin + */ + public static final String MONTH_BEGIN = "month_begin"; + /** + * add_months + */ + public static final String ADD_MONTHS = "add_months"; + /** + * month_end + */ + public static final String MONTH_END = "month_end"; + /** + * week_begin + */ + public static final String WEEK_BEGIN = "week_begin"; + /** + * week_end + */ + public static final String WEEK_END = "week_end"; + /** + * this_day + */ + public static final String THIS_DAY = "this_day"; + /** + * last_day + */ + public static final String LAST_DAY = "last_day"; + + /** + * month_first_day + */ + public static final String MONTH_FIRST_DAY = "month_first_day"; + + /** + * month_last_day + */ + public static final String MONTH_LAST_DAY = "month_last_day"; + + /** + * week_first_day + */ + public static final String WEEK_FIRST_DAY = "week_first_day"; + + /** + * week_last_day + */ + public static final String WEEK_LAST_DAY = "week_last_day"; + + /** + * year_week + */ + public static final String YEAR_WEEK = "year_week"; + /** + * timestamp + */ + public static final String TIMESTAMP = "timestamp"; + public static final char SUBTRACT_CHAR = '-'; + public static final char ADD_CHAR = '+'; + public static final char MULTIPLY_CHAR = '*'; + public static final char DIVISION_CHAR = '/'; + public static final char LEFT_BRACE_CHAR = '('; + public static final char RIGHT_BRACE_CHAR = ')'; + public static final String ADD_STRING = "+"; + public static final String MULTIPLY_STRING = "*"; + public static final String DIVISION_STRING = "/"; + public static final String LEFT_BRACE_STRING = "("; + public static final char P = 'P'; + public static final char N = 'N'; + public static final String SUBTRACT_STRING = "-"; + public static final String LOCAL_PARAMS_LIST = "localParamsList"; + public static final String TASK_TYPE = "taskType"; + public static final String QUEUE = "queue"; + /** + * default display rows + */ + public static final int DEFAULT_DISPLAY_ROWS = 10; + + /** + * jar + */ + public static final String JAR = "jar"; + + /** + * hadoop + */ + public static final String HADOOP = "hadoop"; + + /** + * -D = + */ + public static final String D = "-D"; + + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; + + /** + * kerberos + */ + public static final String KERBEROS = "kerberos"; + + /** + * kerberos expire time + */ + public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time"; + + /** + * java.security.krb5.conf + */ + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + /** + * java.security.krb5.conf.path + */ + public static final String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; + + /** + * loginUserFromKeytab user + */ + public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; + + /** + * loginUserFromKeytab path + */ + public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = + "hadoop.security.authentication.startup.state"; + + /** + * hdfs/s3 configuration + * resource.storage.upload.base.path + */ + public static final String RESOURCE_UPLOAD_PATH = "resource.storage.upload.base.path"; + + /** + * data.quality.jar.dir + */ + public static final String DATA_QUALITY_JAR_DIR = "data-quality.jar.dir"; + + public static final String TASK_TYPE_DATA_QUALITY = "DATA_QUALITY"; + + public static final Set TASK_TYPE_SET_K8S = Sets.newHashSet("K8S", "KUBEFLOW"); + + /** + * azure config + */ + public static final String AZURE_CLIENT_ID = "resource.azure.client.id"; + public static final String AZURE_CLIENT_SECRET = "resource.azure.client.secret"; + public static final String AZURE_ACCESS_SUB_ID = "resource.azure.subId"; + public static final String AZURE_SECRET_TENANT_ID = "resource.azure.tenant.id"; + public static final String QUERY_INTERVAL = "resource.query.interval"; + + /** + * use for k8s task + */ + public static final String API_VERSION = "batch/v1"; + public static final String RESTART_POLICY = "Never"; + public static final String MEMORY = "memory"; + public static final String CPU = "cpu"; + public static final String LAYER_LABEL = "k8s.cn/layer"; + public static final String LAYER_LABEL_VALUE = "batch"; + public static final String NAME_LABEL = "k8s.cn/name"; + public static final String TASK_INSTANCE_ID = "taskInstanceId"; + public static final String MI = "Mi"; + public static final int JOB_TTL_SECONDS = 300; + public static final int LOG_LINES = 500; + public static final String NAMESPACE_NAME = "name"; + public static final String CLUSTER = "cluster"; + + /** + * spark / flink on k8s label name + */ + public static final String UNIQUE_LABEL_NAME = "dolphinscheduler-label"; + + /** + * conda config used by jupyter task plugin + */ + public static final String CONDA_PATH = "conda.path"; + + // Loop task constants + public static final Duration DEFAULT_LOOP_STATUS_INTERVAL = Duration.ofSeconds(5L); + + /** + * sql params regex + */ + public static final String GROUP_NAME1 = "paramName1"; + public static final String GROUP_NAME2 = "paramName2"; + public static final String SQL_PARAMS_REGEX = + String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", GROUP_NAME1, GROUP_NAME2); + public static final Pattern SQL_PARAMS_PATTERN = Pattern.compile(SQL_PARAMS_REGEX); + + public static final String AZURE_SQL_DATABASE_SPN = "https://database.windows.net/"; + public static final String AZURE_SQL_DATABASE_TOKEN_SCOPE = "/.default"; + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java index 7223fe62a387..417af00b3de7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceClientProvider.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.api.plugin; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.spi.datasource.AdHocDataSourceClient; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; @@ -47,7 +47,8 @@ public class DataSourceClientProvider { // One DataSourceClient corresponds to one unique datasource. private static final Cache POOLED_DATASOURCE_CLIENT_CACHE = CacheBuilder.newBuilder() - .expireAfterWrite(PropertyUtils.getLong(TaskConstants.KERBEROS_EXPIRE_TIME, 24L), TimeUnit.HOURS) + .expireAfterWrite(PropertyUtils.getLong(DataSourceConstants.KERBEROS_EXPIRE_TIME, 24L), + TimeUnit.HOURS) .removalListener((RemovalListener) notification -> { try (PooledDataSourceClient closedClient = notification.getValue()) { log.info("Datasource: {} is removed from cache due to expire", notification.getKey()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JDBCDataSourceProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JDBCDataSourceProvider.java index d4651b224d44..7aae4ec5af63 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JDBCDataSourceProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JDBCDataSourceProvider.java @@ -17,9 +17,8 @@ package org.apache.dolphinscheduler.plugin.datasource.api.provider; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; @@ -54,7 +53,7 @@ public static HikariDataSource createOneSessionJdbcDataSource(BaseConnectionPara dataSource.setUsername(properties.getUser()); dataSource.setPassword(PasswordUtils.decodePassword(properties.getPassword())); - Boolean isOneSession = PropertyUtils.getBoolean(Constants.SUPPORT_HIVE_ONE_SESSION, false); + Boolean isOneSession = PropertyUtils.getBoolean(DataSourceConstants.SUPPORT_HIVE_ONE_SESSION, false); dataSource.setMinimumIdle( isOneSession ? 1 : PropertyUtils.getInt(DataSourceConstants.SPRING_DATASOURCE_MIN_IDLE, 5)); dataSource.setMaximumPoolSize( diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java index eb8f2ddef05e..04bd243c0e03 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java @@ -18,15 +18,15 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_STORAGE_TYPE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATA_QUALITY_JAR_DIR; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF_PATH; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.KERBEROS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_USER_KEY_TAB_PATH; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_USER_KEY_TAB_USERNAME; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESOURCE_UPLOAD_PATH; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.DATA_QUALITY_JAR_DIR; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.JAVA_SECURITY_KRB5_CONF; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.JAVA_SECURITY_KRB5_CONF_PATH; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.KERBEROS; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.LOGIN_USER_KEY_TAB_PATH; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.LOGIN_USER_KEY_TAB_USERNAME; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.RESOURCE_UPLOAD_PATH; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.StorageType; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java index 779e6171488c..c1eb9f04425d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/PasswordUtils.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATASOURCE_ENCRYPTION_ENABLE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATASOURCE_ENCRYPTION_SALT; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.DATASOURCE_ENCRYPTION_SALT; +import static org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT; import org.apache.dolphinscheduler.common.utils.PropertyUtils; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml index 932cc2af293b..b928713589c8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml @@ -27,16 +27,39 @@ dolphinscheduler-datasource-athena jar + + datasource.athena + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api provided + org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common + ${project.version} + provided - + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java index c0ff56da6775..13f128bc190c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.athena.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/pom.xml index 9e36f2ea0783..d5d99c6ab104 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/pom.xml @@ -28,16 +28,23 @@ jar ${project.artifactId} + + datasource.azure.sql + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided + org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -56,4 +63,21 @@ azure-identity + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/param/AzureSQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/param/AzureSQLDataSourceProcessor.java index 462ceca36c5d..10f7d32ad585 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/param/AzureSQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/param/AzureSQLDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.azuresql.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml index f1c3e5885db4..eef74c330067 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml @@ -28,16 +28,23 @@ jar ${project.artifactId} + + datasource.clickhouse + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -68,4 +75,21 @@ lz4-java + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java index 81a4415f5d15..8f5959d25202 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.clickhouse.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/pom.xml index f965b1df8f70..061fc3846165 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/pom.xml @@ -28,24 +28,45 @@ jar ${project.artifactId} - + + datasource.dameng + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided com.dameng DmJdbcDriver18 - + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java index bc31bdcd492b..b7d002abd171 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.dameng.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml index 91478998b073..41db8a745e64 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/pom.xml @@ -28,16 +28,23 @@ jar ${project.artifactId} + + datasource.databend + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -68,4 +75,21 @@ lz4-java + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java index e2197b7de778..c2917404ca56 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-databend/src/main/java/org/apache/dolphinscheduler/plugin/datasource/databend/param/DatabendDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.databend.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml index c596351918dd..5652b0eca537 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/pom.xml @@ -28,17 +28,40 @@ jar ${project.artifactId} + + datasource.db2 + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java index 1d7c448355e2..d9aad8ce5520 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.db2.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml index a0d31b23e8e7..6b9a3d66ce3d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/pom.xml @@ -28,14 +28,22 @@ jar ${project.artifactId} + + datasource.doris + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + provided + org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common + ${project.version} + provided @@ -43,4 +51,21 @@ mysql-connector-java + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java index c745ac77de41..e4322e3beed6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.doris.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/pom.xml index e35249d9623b..8868f4219630 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.hana + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -46,4 +52,21 @@ ngdbc + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/param/HanaDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/param/HanaDataSourceProcessor.java index e6f59aece8db..41eadaef9fd1 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/param/HanaDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hana/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hana/param/HanaDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.hana.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml index bcc11a66181b..a0e463d0b787 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml @@ -28,18 +28,23 @@ jar ${project.artifactId} - + + datasource.hive + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -323,4 +328,21 @@ + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java index 09d9f4b9630d..d58f7efdb7dd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java index a39cdaf61436..8e9469c16921 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.security; -import static org.apache.dolphinscheduler.common.constants.Constants.JAVA_SECURITY_KRB5_CONF; - import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.StorageType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; @@ -100,7 +98,7 @@ private static UserGroupInformation createKerberosUser() { String keytab = PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH); String principal = PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME); if (StringUtils.isNotBlank(krb5File)) { - System.setProperty(JAVA_SECURITY_KRB5_CONF, krb5File); + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, krb5File); } Configuration hadoopConf = new Configuration(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml index 2d51b85fed62..26dece7484ee 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.k8s + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -47,4 +53,20 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/pom.xml index 86381d7aadfe..aeb6f95ac6a3 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/pom.xml @@ -27,23 +27,23 @@ dolphinscheduler-datasource-kyuubi jar ${project.artifactId} - - - org.apache.dolphinscheduler - dolphinscheduler-spi - provided - + + datasource.kyuubi + + org.apache.dolphinscheduler dolphinscheduler-datasource-api ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-common + ${project.version} provided @@ -53,4 +53,20 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessor.java index 760153d7184a..72129b87acb8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-kyuubi/src/main/java/org/apache/dolphinscheduler/plugin/datasource/kyuubi/param/KyuubiDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.kyuubi.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml index 8c996cb2dfe3..ad4653953857 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.mysql + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -46,4 +52,21 @@ mysql-connector-java + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java index 0c93b9821211..fe0687390158 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.mysql.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/pom.xml index 37e228c9e688..1830f45bf527 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/pom.xml @@ -28,16 +28,40 @@ jar ${project.artifactId} + + datasource.oceanbase + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided + org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java index 4d89f28eab1c..8dda78af5d56 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oceanbase/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oceanbase/param/OceanBaseDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.oceanbase.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml index ac8ac1ae6acb..c7d606c8bb53 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.oracle + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -46,4 +52,21 @@ ojdbc8 + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java index 5f24b8cc1939..106ea46a202c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.oracle.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml index d7e379c81a9e..cfcab13a12e9 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.postgresql + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -46,4 +52,21 @@ postgresql + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java index 28a0aa294578..260e5cb4afb7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.postgresql.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml index a0942ea86b04..c8aaec25a43d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/pom.xml @@ -28,15 +28,22 @@ jar ${project.artifactId} + + datasource.presto + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common + ${project.version} + provided @@ -44,4 +51,21 @@ presto-jdbc + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java index e466ea98c9f3..f72bdb2b640f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.presto.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml index 850aef34d1c9..ce50adbb8d77 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml @@ -26,26 +26,50 @@ dolphinscheduler-datasource-redshift jar - + + datasource.redshift + + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common + ${project.version} + provided + com.amazon.redshift redshift-jdbc42 + com.amazonaws aws-java-sdk-redshift - + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java index c9163fb5df0e..6535ac45d7ca 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml index 870abed97ad4..a0ce1477bbd8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.sagemaker + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -47,4 +53,20 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/pom.xml index 26cb50c88415..eddca17ca8c7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/pom.xml @@ -28,18 +28,23 @@ jar ${project.artifactId} - + + datasource.snowflake + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -47,4 +52,21 @@ snowflake-jdbc + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDatasourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDatasourceProcessor.java index 02229ed6b3db..764bc8e34557 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDatasourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-snowflake/src/main/java/org/apache/dolphinscheduler/plugin/datasource/snowflake/param/SnowflakeDatasourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.snowflake.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; @@ -37,6 +37,7 @@ import lombok.extern.slf4j.Slf4j; import com.google.auto.service.AutoService; + @AutoService(DataSourceProcessor.class) @Slf4j public class SnowflakeDatasourceProcessor extends AbstractDataSourceProcessor { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml index 2f954cb77c51..923af62aff51 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/pom.xml @@ -28,29 +28,46 @@ jar ${project.artifactId} + + datasource.spark + + org.apache.dolphinscheduler - dolphinscheduler-spi - provided - - - org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-hive + dolphinscheduler-common ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-datasource-hive ${project.version} + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java index 69260b29c99f..b34ff17ba75a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.spark.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml index 6e0d48cbf805..dd53ecea947e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/pom.xml @@ -28,16 +28,23 @@ jar ${project.artifactId} + + datasource.sqlserver + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided + org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -51,4 +58,21 @@ + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java index 687d853fffae..bc34a309b135 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/pom.xml index 6645ee8a9057..2ce7810c2cf3 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/pom.xml @@ -28,20 +28,45 @@ jar ${project.artifactId} + + datasource.ssh + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided + org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided + org.apache.sshd sshd-scp + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/pom.xml index 7f1410c99b32..d4e9c7d93c04 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/pom.xml @@ -26,11 +26,23 @@ dolphinscheduler-datasource-starrocks + + datasource.starrocks + + org.apache.dolphinscheduler dolphinscheduler-datasource-api ${project.version} + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided @@ -39,4 +51,20 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/param/StarRocksDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/param/StarRocksDataSourceProcessor.java index ec76899d59c6..bab10b59ea4a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/param/StarRocksDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-starrocks/src/main/java/org/apache/dolphinscheduler/plugin/datasource/starrocks/param/StarRocksDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.starrocks.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/pom.xml index 45cc19d3f57e..308c9663f8a6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/pom.xml @@ -28,15 +28,22 @@ jar ${project.artifactId} + + datasource.trino + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common + ${project.version} + provided @@ -44,4 +51,21 @@ trino-jdbc + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java index bd53acaaf8fd..e7177c6b9558 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-trino/src/main/java/org/apache/dolphinscheduler/plugin/datasource/trino/param/TrinoDataSourceProcessor.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.datasource.trino.param; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/pom.xml index 47f04db5884d..3b4aff1e8f53 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/pom.xml @@ -28,18 +28,23 @@ jar ${project.artifactId} - + + datasource.vertica + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -47,4 +52,21 @@ vertica-jdbc + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/param/VerticaDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/param/VerticaDataSourceProcessor.java index 9505ab363dab..e811eaea173c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/param/VerticaDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-vertica/src/main/java/org/apache/dolphinscheduler/plugin/datasource/vertica/param/VerticaDataSourceProcessor.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.datasource.vertica.param; -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml index 90c7f1ece5aa..6d64a3ae29f0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml @@ -28,17 +28,23 @@ jar ${project.artifactId} + + datasource.zeppelin + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-datasource-api + ${project.version} provided org.apache.dolphinscheduler - dolphinscheduler-datasource-api + dolphinscheduler-common ${project.version} + provided @@ -47,4 +53,20 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 1f712364d968..313b3cdea7b8 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -71,10 +71,4 @@ - - - org.apache.dolphinscheduler - dolphinscheduler-common - - diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index ee4c85589ed3..046b5335183f 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -26,9 +26,6 @@ dolphinscheduler-dist ${project.artifactId} - - true - @@ -43,7 +40,6 @@ - org.apache.dolphinscheduler dolphinscheduler-standalone-server @@ -118,6 +114,81 @@ + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + assembly-plugins + + exec + + package + + bash + ${project.basedir} + + src/main/assembly/assembly-plugins.sh + + + + + + + + + + docker + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + docker-build + + exec + + package + + ${docker.build.skip} + + 1 + + bash + ${project.basedir} + + src/main/docker/docker-build.sh + ${docker.hub} + ${docker.tag} + + + + + docker-push + + exec + + deploy + + ${docker.push.skip} + + 1 + + docker + ${project.basedir} + + src/main/docker/docker-push.sh + ${docker.hub} + ${docker.tag} + + + + + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-json-path.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-json-path.txt deleted file mode 100644 index 70eda5041a12..000000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-json-path.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 Jayway - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh b/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh new file mode 100755 index 000000000000..36cb430df2bb --- /dev/null +++ b/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -xeo pipefail + +DIST_DIR="$(pwd)/target" +BIN_TAR_FILE="$DIST_DIR/apache-dolphinscheduler-*-bin.tar.gz" +if [ ! -f $BIN_TAR_FILE ]; then + echo "$BIN_TAR_FILE not found!!!" + exit 1 +fi + +cd $DIST_DIR && tar -zxf apache-dolphinscheduler-*-bin.tar.gz +cd $DIST_DIR/apache-dolphinscheduler-*-bin +BIN_DIR=$(pwd) + +# move *-plugins/target/*-plugin/target/*.jar to *-plugins/ +PLUGINS_PATH=( +alert-plugins +datasource-plugins +storage-plugins +task-plugins +) + +for plugin_path in ${PLUGINS_PATH[@]} +do + cd $BIN_DIR/plugins/$plugin_path + find ./* -name "*.jar" | xargs -I {} mv {} ./ + ls -d */ | xargs -I {} rm -rf {} +done + +# move *-server/libs/*.jar to libs/ and create symbolic link in *-server/libs/ +MODULES_PATH=( +api-server +master-server +worker-server +alert-server +tools +) + +SHARED_LIB_DIR="$BIN_DIR/libs" +mkdir -p $SHARED_LIB_DIR + +for module in ${MODULES_PATH[@]} +do + MODULE_LIB_DIR="$BIN_DIR/$module/libs" + cd $MODULE_LIB_DIR + for jar in $(find $MODULE_LIB_DIR/* -name "*.jar" -execdir echo {} ';'); do + # move jar file to share lib directory + mv $MODULE_LIB_DIR/$jar $SHARED_LIB_DIR/$jar + + # create a symbolic link in the subproject's lib directory + ln -s ../../libs/$jar $jar + done +done + +# repack bin tar +BIN_TAR_FILE_NAME=$(basename $BIN_TAR_FILE) +cd $DIST_DIR && tar -zcf $BIN_TAR_FILE_NAME apache-dolphinscheduler-*-bin + +echo "assembly-plugins.sh done" diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml index a90dd9ce093a..72018cfb9ef8 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml @@ -46,21 +46,43 @@ worker-server + - ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server - standalone-server - - libs/ - bin/ - + ${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-plugins + + dolphinscheduler-alert-*/target/dolphinscheduler-alert-*-shade.jar + + plugins/alert-plugins + + + + ${basedir}/../dolphinscheduler-datasource-plugin + + dolphinscheduler-datasource-*/target/dolphinscheduler-datasource-*-shade.jar + + plugins/datasource-plugins + + + + ${basedir}/../dolphinscheduler-task-plugin + + dolphinscheduler-task-*/target/dolphinscheduler-task-*-shade.jar + + plugins/task-plugins + - ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server/libs/standalone-server - standalone-server/libs/standalone-server + ${basedir}/../dolphinscheduler-storage-plugin + + dolphinscheduler-storage-*/target/dolphinscheduler-storage-*-shade.jar + + plugins/storage-plugins + + - ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server/bin - standalone-server/bin + ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server + standalone-server diff --git a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile b/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile similarity index 79% rename from dolphinscheduler-standalone-server/src/main/docker/Dockerfile rename to dolphinscheduler-dist/src/main/docker/alert-server.dockerfile index 6eebf041622c..cc8d0646c4dc 100644 --- a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile +++ b/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile @@ -17,10 +17,9 @@ FROM eclipse-temurin:8-jdk -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -ENV DATA_QUALITY_JAR_DIR /opt/dolphinscheduler/libs/worker-server +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler RUN apt update ; \ apt install -y sudo ; \ @@ -28,8 +27,8 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/standalone-server $DOLPHINSCHEDULER_HOME +ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME EXPOSE 12345 25333 -CMD [ "/bin/bash", "./bin/start.sh" ] +CMD [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] diff --git a/dolphinscheduler-worker/src/main/docker/Dockerfile b/dolphinscheduler-dist/src/main/docker/api-server.dockerfile similarity index 79% rename from dolphinscheduler-worker/src/main/docker/Dockerfile rename to dolphinscheduler-dist/src/main/docker/api-server.dockerfile index 38e8bd6a33a5..a0201810d2e0 100644 --- a/dolphinscheduler-worker/src/main/docker/Dockerfile +++ b/dolphinscheduler-dist/src/main/docker/api-server.dockerfile @@ -17,9 +17,9 @@ FROM eclipse-temurin:8-jdk -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler RUN apt update ; \ apt install -y sudo ; \ @@ -27,8 +27,8 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/worker-server $DOLPHINSCHEDULER_HOME +ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME -EXPOSE 1235 +EXPOSE 12345 25333 -CMD [ "/bin/bash", "./bin/start.sh" ] +CMD [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] diff --git a/dolphinscheduler-dist/src/main/docker/docker-build.sh b/dolphinscheduler-dist/src/main/docker/docker-build.sh new file mode 100644 index 000000000000..15a946101480 --- /dev/null +++ b/dolphinscheduler-dist/src/main/docker/docker-build.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -xeo pipefail + +DOCKER_HUB=$1 +DOCKER_TAG=$2 +DOCKER_REPO_BASE=dolphinscheduler + +CURRENT_HOME=$(dirname $(readlink -f "$0")) + +docker buildx build --load --no-cache -t $DOCKER_HUB/$DOCKER_REPO_BASE-api:$DOCKER_TAG -t $DOCKER_HUB/$DOCKER_REPO_BASE-api:latest -f ${CURRENT_HOME}/api-server.dockerfile . +docker buildx build --load -t $DOCKER_HUB/$DOCKER_REPO_BASE-master:$DOCKER_TAG -t $DOCKER_HUB/$DOCKER_REPO_BASE-master:latest -f ${CURRENT_HOME}/master-server.dockerfile . +docker buildx build --load -t $DOCKER_HUB/$DOCKER_REPO_BASE-worker:$DOCKER_TAG -t $DOCKER_HUB/$DOCKER_REPO_BASE-worker:latest -f ${CURRENT_HOME}/worker-server.dockerfile . +docker buildx build --load -t $DOCKER_HUB/$DOCKER_REPO_BASE-alert-server:$DOCKER_TAG -t $DOCKER_HUB/$DOCKER_REPO_BASE-alert-server:latest -f ${CURRENT_HOME}/alert-server.dockerfile . +docker buildx build --load -t $DOCKER_HUB/$DOCKER_REPO_BASE-standalone-server:$DOCKER_TAG -t $DOCKER_HUB/$DOCKER_REPO_BASE-standalone-server:latest -f ${CURRENT_HOME}/standalone-server.dockerfile . +docker buildx build --load -t $DOCKER_HUB/$DOCKER_REPO_BASE-tools:$DOCKER_TAG -t $DOCKER_HUB/$DOCKER_REPO_BASE-tools:latest -f ${CURRENT_HOME}/tools.dockerfile . diff --git a/dolphinscheduler-dist/src/main/docker/docker-push.sh b/dolphinscheduler-dist/src/main/docker/docker-push.sh new file mode 100644 index 000000000000..e171f8a7c2b3 --- /dev/null +++ b/dolphinscheduler-dist/src/main/docker/docker-push.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -xeo pipefail + +DOCKER_HUB=$1 +DOCKER_TAG=$2 +DOCKER_REPO_BASE=dolphinscheduler + +CURRENT_HOME=$(dirname $(readlink -f "$0")) + +docker buildx build --push --no-cache --platform linux/amd64,linux/arm64 -t $DOCKER_HUB/$DOCKER_REPO_BASE-api:$DOCKER_TAG -f ${CURRENT_HOME}/api-server.dockerfile . +docker buildx build --push --platform linux/amd64,linux/arm64 -t $DOCKER_HUB/$DOCKER_REPO_BASE-master:$DOCKER_TAG -f ${CURRENT_HOME}/master-server.dockerfile . +docker buildx build --push --platform linux/amd64,linux/arm64 -t $DOCKER_HUB/$DOCKER_REPO_BASE-worker:$DOCKER_TAG -f ${CURRENT_HOME}/worker-server.dockerfile . +docker buildx build --push --platform linux/amd64,linux/arm64 -t $DOCKER_HUB/$DOCKER_REPO_BASE-alert-server:$DOCKER_TAG -f ${CURRENT_HOME}/alert-server.dockerfile . +docker buildx build --push --platform linux/amd64,linux/arm64 -t $DOCKER_HUB/$DOCKER_REPO_BASE-standalone-server:$DOCKER_TAG -f ${CURRENT_HOME}/standalone-server.dockerfile . +docker buildx build --push --platform linux/amd64,linux/arm64 -t $DOCKER_HUB/$DOCKER_REPO_BASE-tools:$DOCKER_TAG -f ${CURRENT_HOME}/tools.dockerfile . diff --git a/dolphinscheduler-api/src/main/docker/Dockerfile b/dolphinscheduler-dist/src/main/docker/master-server.dockerfile similarity index 71% rename from dolphinscheduler-api/src/main/docker/Dockerfile rename to dolphinscheduler-dist/src/main/docker/master-server.dockerfile index 8b7f1c7f16c5..fbcc8bf62112 100644 --- a/dolphinscheduler-api/src/main/docker/Dockerfile +++ b/dolphinscheduler-dist/src/main/docker/master-server.dockerfile @@ -15,16 +15,20 @@ # limitations under the License. # -FROM eclipse-temurin:8-jre +FROM eclipse-temurin:8-jdk -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/api-server $DOLPHINSCHEDULER_HOME +ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME EXPOSE 12345 25333 -CMD [ "/bin/bash", "./bin/start.sh" ] +CMD [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] diff --git a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile new file mode 100644 index 000000000000..b0cad0632a31 --- /dev/null +++ b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jdk + +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME + +EXPOSE 12345 25333 + +CMD [ "/bin/bash", "/opt/dolphinscheduler/standalone-server/bin/start.sh" ] diff --git a/dolphinscheduler-tools/src/main/docker/Dockerfile b/dolphinscheduler-dist/src/main/docker/tools.dockerfile similarity index 78% rename from dolphinscheduler-tools/src/main/docker/Dockerfile rename to dolphinscheduler-dist/src/main/docker/tools.dockerfile index 5bddf868174c..c795a2b5074a 100644 --- a/dolphinscheduler-tools/src/main/docker/Dockerfile +++ b/dolphinscheduler-dist/src/main/docker/tools.dockerfile @@ -15,15 +15,19 @@ # limitations under the License. # -FROM eclipse-temurin:8-jre +FROM eclipse-temurin:8-jdk -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* WORKDIR $DOLPHINSCHEDULER_HOME # see doc: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/upgrade.html -ADD ./target/tools $DOLPHINSCHEDULER_HOME/tools +ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME ENTRYPOINT [ "/bin/bash" ] diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile b/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile similarity index 70% rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile rename to dolphinscheduler-dist/src/main/docker/worker-server.dockerfile index e5256ce32641..c8ce0b61c98c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile +++ b/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile @@ -15,16 +15,20 @@ # limitations under the License. # -FROM eclipse-temurin:8-jre +FROM eclipse-temurin:8-jdk -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/alert-server $DOLPHINSCHEDULER_HOME +ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME -EXPOSE 50052 50053 +EXPOSE 12345 25333 -CMD [ "/bin/bash", "./bin/start.sh" ] +CMD [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml index 8287eb83de05..7a3599531069 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml @@ -29,7 +29,7 @@ services: - e2e volumes: - ./download-mysql.sh:/tmp/download-mysql.sh - entrypoint: ['bash', '-c', '/bin/bash /tmp/download-mysql.sh && /opt/dolphinscheduler/bin/start.sh && tail -f /dev/null'] + entrypoint: ['bash', '-c', '/bin/bash /tmp/download-mysql.sh && /bin/bash /opt/dolphinscheduler/standalone-server/bin/start.sh && tail -f /dev/null'] healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/download-mysql.sh b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/download-mysql.sh index ef0ea7309c00..ff848739cd55 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/download-mysql.sh +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/download-mysql.sh @@ -17,7 +17,7 @@ set -ex -DS_HOME=/opt/dolphinscheduler/libs/standalone-server +DS_HOME=/opt/dolphinscheduler/libs MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" MYSQL_DRIVER="mysql-connector-java-8.0.16.jar" diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml index 49b9de813fa8..46d24d55e978 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml @@ -33,8 +33,8 @@ services: timeout: 5s retries: 120 volumes: - - ./common.properties:/opt/dolphinscheduler/conf/common.properties - - ./aws.yaml:/opt/dolphinscheduler/conf/aws.yaml + - ./common.properties:/opt/dolphinscheduler/standalone-server/conf/common.properties + - ./aws.yaml:/opt/dolphinscheduler/standalone-server/conf/aws.yaml depends_on: s3: condition: service_healthy diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 5205b04add2b..1f3c706b8516 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -304,6 +304,11 @@ + + org.javadelight + delight-nashorn-sandbox + + org.springframework.cloud @@ -356,18 +361,4 @@ - - - - docker - - - - org.codehaus.mojo - exec-maven-plugin - - - - - diff --git a/dolphinscheduler-master/src/main/bin/start.sh b/dolphinscheduler-master/src/main/bin/start.sh index d05d70a6376d..74b7ba1112e1 100644 --- a/dolphinscheduler-master/src/main/bin/start.sh +++ b/dolphinscheduler-master/src/main/bin/start.sh @@ -18,11 +18,12 @@ set -eo pipefail BIN_DIR=$(dirname $(readlink -f "$0")) -DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd) +DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/../..;pwd) +MASTER_HOME=$(cd ${BIN_DIR}/..;pwd) -source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" +source "$MASTER_HOME/conf/dolphinscheduler_env.sh" -JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh +JVM_ARGS_ENV_FILE=${MASTER_HOME}/bin/jvm_args_env.sh JVM_ARGS="-server" if [ -f $JVM_ARGS_ENV_FILE ]; then @@ -37,12 +38,31 @@ fi JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then - JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" + JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport -DDOCKER=true" fi echo "JAVA_HOME=${JAVA_HOME}" echo "JAVA_OPTS=${JAVA_OPTS}" +MODULES_PATH=( +master-server +) + +CP="" +for module in ${MODULES_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/$module/libs/*" +done + +PLUGINS_PATH=( +datasource-plugins +storage-plugins +task-plugins +) + +for plugin in ${PLUGINS_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" +done + $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ + -cp "$MASTER_HOME/conf""$CP" \ org.apache.dolphinscheduler.server.master.MasterServer diff --git a/dolphinscheduler-master/src/main/docker/Dockerfile b/dolphinscheduler-master/src/main/docker/Dockerfile deleted file mode 100644 index 6699eb9d205b..000000000000 --- a/dolphinscheduler-master/src/main/docker/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM eclipse-temurin:8-jre - -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler - -WORKDIR $DOLPHINSCHEDULER_HOME - -ADD ./target/master-server $DOLPHINSCHEDULER_HOME - -EXPOSE 5678 5679 - -CMD [ "/bin/bash", "./bin/start.sh" ] diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml index aa592b9da450..a11724a26191 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml @@ -32,10 +32,12 @@ org.apache.dolphinscheduler dolphinscheduler-registry-api + ${project.version} org.apache.dolphinscheduler dolphinscheduler-common + ${project.version} com.zaxxer diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml index b65e1d13406d..6bf1250a7956 100644 --- a/dolphinscheduler-spi/pom.xml +++ b/dolphinscheduler-spi/pom.xml @@ -39,37 +39,17 @@ - - org.apache.dolphinscheduler - dolphinscheduler-common - - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - org.slf4j slf4j-api + - com.baomidou - mybatis-plus-annotation + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} provided - - com.google.guava - guava - diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java index 72b0322da75d..af529a2b31a6 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbConnectType.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.spi.enums; +import lombok.Getter; + +@Getter public enum DbConnectType { ORACLE_SERVICE_NAME(0, "Oracle Service Name"), @@ -31,12 +34,4 @@ public enum DbConnectType { private final String descp; - public int getCode() { - return code; - } - - public String getDescp() { - return descp; - } - } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 360e788cb3f1..639f8d573983 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -23,9 +23,12 @@ import java.util.Map; import java.util.NoSuchElementException; +import lombok.Getter; + import com.baomidou.mybatisplus.annotation.EnumValue; import com.google.common.base.Functions; +@Getter public enum DbType { MYSQL(0, "mysql", "mysql"), @@ -84,18 +87,6 @@ public static DbType ofName(String name) { .orElseThrow(() -> new NoSuchElementException("no such db type")); } - public int getCode() { - return code; - } - - public String getName() { - return name; - } - - public String getDescp() { - return descp; - } - public boolean isHive() { return this == DbType.HIVE; } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java index 242b3cdc42f5..cfc89f8c2469 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.spi.enums; +import lombok.Getter; + /** * have_script * have_file @@ -25,6 +27,7 @@ * have_map_variables * have_alert */ +@Getter public enum Flag { /** @@ -42,11 +45,4 @@ public enum Flag { private final int code; private final String descp; - public int getCode() { - return code; - } - - public String getDescp() { - return descp; - } } diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java index d058943d4972..76ed587dfc3d 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.spi.params; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.spi.params.base.DataType; import org.apache.dolphinscheduler.spi.params.base.ParamsOptions; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -32,9 +33,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; - /** * PluginParamsTransfer Tester. */ @@ -186,9 +184,9 @@ public void testGetParamsJson() { + ",\"disabled\":false},{\"label\":\"text\",\"value\":\"text\",\"disabled\":false},{\"label\"" + ":\"attachment\",\"value\":\"attachment\",\"disabled\":false},{\"label\":\"tableattachment\"" + ",\"value\":\"tableattachment\",\"disabled\":false}]}]"; - JsonElement paramsJsonElement = JsonParser.parseString(paramsJson); - JsonElement paramsAssertJsonElement = JsonParser.parseString(paramsJsonAssert); - Assertions.assertEquals(paramsAssertJsonElement, paramsJsonElement); + String paramsJsonElement = JSONUtils.toJsonString(paramsJson); + String paramsAssertJsonElement = JSONUtils.toJsonString(paramsJsonAssert); + Assertions.assertEquals(paramsJsonElement, paramsAssertJsonElement); } @Test diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml index 921eb6bf8c7c..54d7b589a4c1 100644 --- a/dolphinscheduler-standalone-server/pom.xml +++ b/dolphinscheduler-standalone-server/pom.xml @@ -42,34 +42,22 @@ org.apache.dolphinscheduler dolphinscheduler-master + provided org.apache.dolphinscheduler dolphinscheduler-worker + provided org.apache.dolphinscheduler dolphinscheduler-api + provided org.apache.dolphinscheduler dolphinscheduler-alert-server - - - - org.apache.curator - curator-test - - - org.javassist - javassist - - - - - - org.springframework.cloud - spring-cloud-starter-kubernetes-client-config + provided @@ -106,18 +94,4 @@ - - - - docker - - - - org.codehaus.mojo - exec-maven-plugin - - - - - diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml index a2b115f90f12..fb70fb31ad0a 100644 --- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml +++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml @@ -25,23 +25,6 @@ false standalone-server - - ${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/target/alert-server/libs - libs/alert-server - - - ${basedir}/../dolphinscheduler-api/target/api-server/libs - libs/api-server - - - ${basedir}/../dolphinscheduler-master/target/master-server/libs - libs/master-server - - - ${basedir}/../dolphinscheduler-worker/target/worker-server/libs - libs/worker-server - - ${basedir}/src/main/resources @@ -110,7 +93,7 @@ false - libs/standalone-server + libs diff --git a/dolphinscheduler-standalone-server/src/main/bin/start.sh b/dolphinscheduler-standalone-server/src/main/bin/start.sh index 03c5196d87da..39596fc69c9a 100755 --- a/dolphinscheduler-standalone-server/src/main/bin/start.sh +++ b/dolphinscheduler-standalone-server/src/main/bin/start.sh @@ -18,12 +18,13 @@ set -eo pipefail BIN_DIR=$(dirname $(readlink -f "$0")) -DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd) +DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/../..;pwd) +STANDALONE_HOME=$(cd ${BIN_DIR}/..;pwd) export DATABASE=${DATABASE:-h2} -source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" +source "$STANDALONE_HOME/conf/dolphinscheduler_env.sh" -JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh +JVM_ARGS_ENV_FILE=${STANDALONE_HOME}/bin/jvm_args_env.sh JVM_ARGS="-server" if [ -f $JVM_ARGS_ENV_FILE ]; then @@ -38,14 +39,39 @@ fi JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then - JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" + JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport -DDOCKER=true" fi +echo "JAVA_HOME=${JAVA_HOME}" +echo "JAVA_OPTS=${JAVA_OPTS}" + +MODULES_PATH=( +api-server +master-server +worker-server +alert-server +) + CP="" -for d in $DOLPHINSCHEDULER_HOME/libs/*; do - CP=$CP:"$d/*" +for module in ${MODULES_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/$module/libs/*" +done + +PLUGINS_PATH=( +alert-plugins +datasource-plugins +storage-plugins +task-plugins +) + +for plugin in ${PLUGINS_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" +done + +for jar in $(find $STANDALONE_HOME/libs/* -name "*.jar"); do + CP=$CP:"$jar" done $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/conf""$CP" \ + -cp "$STANDALONE_HOME/conf""$CP" \ org.apache.dolphinscheduler.StandaloneServer diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/pom.xml index e35c6d10fdcd..84475e67da64 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/pom.xml @@ -26,22 +26,45 @@ dolphinscheduler-storage-abs - - - com.azure - azure-storage-blob - + + storage.abs + + org.apache.dolphinscheduler dolphinscheduler-storage-api ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-common ${project.version} + provided + + + + com.azure + azure-storage-blob + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java index 81b93ea207fc..0b6030082e0a 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java @@ -20,11 +20,11 @@ import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.commons.lang3.StringUtils; @@ -74,7 +74,7 @@ public AbsStorageOperator(AbsStorageProperties absStorageProperties) { public String getStorageBaseDirectory() { // All directory should end with File.separator if (getStorageBaseDirectory().startsWith("/")) { - log.warn("{} -> {} should not start with / in abs", Constants.RESOURCE_UPLOAD_PATH, + log.warn("{} -> {} should not start with / in abs", StorageConstants.RESOURCE_UPLOAD_PATH, getStorageBaseDirectory()); return getStorageBaseDirectory().substring(1); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java index 6428ec0980a2..f8bc3b23b170 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.storage.abs; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import com.google.auto.service.AutoService; @@ -36,10 +36,10 @@ public StorageOperator createStorageOperate() { private AbsStorageProperties getAbsStorageProperties() { return AbsStorageProperties.builder() - .containerName(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONTAINER_NAME)) - .connectionString(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONNECTION_STRING)) - .storageAccountName(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_ACCOUNT_NAME)) - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .containerName(PropertyUtils.getString(StorageConstants.AZURE_BLOB_STORAGE_CONTAINER_NAME)) + .connectionString(PropertyUtils.getString(StorageConstants.AZURE_BLOB_STORAGE_CONNECTION_STRING)) + .storageAccountName(PropertyUtils.getString(StorageConstants.AZURE_BLOB_STORAGE_ACCOUNT_NAME)) + .resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) .build(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/pom.xml index 5e5311dadc4d..42ad6494bf87 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/pom.xml @@ -32,5 +32,12 @@ dolphinscheduler-spi ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java index 924c581248fc..72e58673597a 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.storage.api; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; @@ -64,7 +64,7 @@ public ResourceMetadata getResourceMetaData(String resourceAbsolutePath) { @Override public String getStorageBaseDirectory() { // All directory should end with File.separator - return PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/tmp/dolphinscheduler"); + return PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/tmp/dolphinscheduler"); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java index 9b30ddb4505a..90631dceee38 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.plugin.storage.api; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_STORAGE_TYPE; +import static org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants.RESOURCE_STORAGE_TYPE; import org.apache.dolphinscheduler.common.utils.PropertyUtils; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java index d8d95be323b2..ef9d58812999 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java @@ -26,7 +26,7 @@ import lombok.Data; import lombok.NoArgsConstructor; -// StorageEneity is an entity representing a resource in the third-part storage service. +// StorageEntity is an entity representing a resource in the third-part storage service. // It is only stored in t_ds_relation_resources_task if the resource is used by a task. // It is not put in the model module because it has more attributes than corresponding objects stored // in table t_ds_relation_resources_task. diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/constants/DateConstants.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/constants/DateConstants.java new file mode 100644 index 000000000000..41473edf5aee --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/constants/DateConstants.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.api.constants; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class DateConstants { + + public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; + + public static final String YYYYMMDDHHMMSSSSS = "yyyyMMddHHmmssSSS"; + + public static final String YYYYMMDD = "yyyyMMdd"; + + /** + * system date(yyyyMMddHHmmss) + */ + public static final String PARAMETER_DATETIME = "system.datetime"; + + /** + * system date(yyyymmdd) today + */ + public static final String PARAMETER_CURRENT_DATE = "system.biz.curdate"; + + /** + * system date(yyyymmdd) yesterday + */ + public static final String PARAMETER_BUSINESS_DATE = "system.biz.date"; + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/constants/StorageConstants.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/constants/StorageConstants.java new file mode 100644 index 000000000000..34189de481f3 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/constants/StorageConstants.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.api.constants; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public final class StorageConstants { + + public static final String RESOURCE_TYPE_FILE = "resources"; + + /** + * resource.hdfs.fs.defaultFS + */ + public static final String FS_DEFAULT_FS = "resource.hdfs.fs.defaultFS"; + + /** + * hdfs defaultFS property name. Should be consistent with the property name in hdfs-site.xml + */ + public static final String HDFS_DEFAULT_FS = "fs.defaultFS"; + + /** + * hdfs configuration + * resource.hdfs.root.user + */ + public static final String HDFS_ROOT_USER = "resource.hdfs.root.user"; + + /** + * hdfs/s3 configuration + * resource.storage.upload.base.path + */ + public static final String RESOURCE_UPLOAD_PATH = "resource.storage.upload.base.path"; + + /** + * resource storage type + */ + public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; + + public static final String AWS_S3_BUCKET_NAME = "aws.s3.bucket.name"; + + public static final String ALIBABA_CLOUD_OSS_BUCKET_NAME = "resource.alibaba.cloud.oss.bucket.name"; + public static final String ALIBABA_CLOUD_OSS_END_POINT = "resource.alibaba.cloud.oss.endpoint"; + + public static final String GOOGLE_CLOUD_STORAGE_BUCKET_NAME = "resource.google.cloud.storage.bucket.name"; + + public static final String GOOGLE_CLOUD_STORAGE_CREDENTIAL = "resource.google.cloud.storage.credential"; + + public static final String AZURE_BLOB_STORAGE_CONNECTION_STRING = "resource.azure.blob.storage.connection.string"; + + public static final String AZURE_BLOB_STORAGE_CONTAINER_NAME = "resource.azure.blob.storage.container.name"; + + public static final String AZURE_BLOB_STORAGE_ACCOUNT_NAME = "resource.azure.blob.storage.account.name"; + + public static final String HUAWEI_CLOUD_ACCESS_KEY_ID = "resource.huawei.cloud.access.key.id"; + public static final String HUAWEI_CLOUD_ACCESS_KEY_SECRET = "resource.huawei.cloud.access.key.secret"; + public static final String HUAWEI_CLOUD_OBS_BUCKET_NAME = "resource.huawei.cloud.obs.bucket.name"; + public static final String HUAWEI_CLOUD_OBS_END_POINT = "resource.huawei.cloud.obs.endpoint"; + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/pom.xml index b193826c834f..3daffaf74620 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/pom.xml @@ -26,7 +26,25 @@ dolphinscheduler-storage-gcs + + storage.gcs + + + + org.apache.dolphinscheduler + dolphinscheduler-storage-api + ${project.version} + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + + com.google.cloud google-cloud-storage @@ -53,17 +71,22 @@ - - - org.apache.dolphinscheduler - dolphinscheduler-storage-api - ${project.version} - - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - ${project.version} - + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java index 00aa746e73eb..45f9c8ab6f52 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java @@ -19,12 +19,12 @@ import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.commons.lang3.StringUtils; @@ -85,7 +85,7 @@ public GcsStorageOperator(GcsStorageProperties gcsStorageProperties) { public String getStorageBaseDirectory() { // All directory should end with File.separator if (resourceBaseAbsolutePath.startsWith("/")) { - log.warn("{} -> {} should not start with / in Gcs", Constants.RESOURCE_UPLOAD_PATH, + log.warn("{} -> {} should not start with / in Gcs", StorageConstants.RESOURCE_UPLOAD_PATH, resourceBaseAbsolutePath); return resourceBaseAbsolutePath.substring(1); } @@ -243,7 +243,7 @@ public StorageEntity getStorageEntity(String resourceAbsolutePath) { private void checkBucketNameExists(String bucketName) { if (StringUtils.isBlank(bucketName)) { - throw new IllegalArgumentException(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME + " is blank"); + throw new IllegalArgumentException(StorageConstants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME + " is blank"); } boolean exist = false; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java index 2bc1a4bfcb81..26eba947218e 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.storage.gcs; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import com.google.auto.service.AutoService; @@ -36,9 +36,9 @@ public StorageOperator createStorageOperate() { public GcsStorageProperties getGcsStorageProperties() { return GcsStorageProperties.builder() - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) - .credential(PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_CREDENTIAL)) - .bucketName(PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME)) + .resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .credential(PropertyUtils.getString(StorageConstants.GOOGLE_CLOUD_STORAGE_CREDENTIAL)) + .bucketName(PropertyUtils.getString(StorageConstants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME)) .build(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml index 04e06df244c2..5551d5eb94bf 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml @@ -26,16 +26,32 @@ dolphinscheduler-storage-hdfs + + storage.hdfs + + org.apache.dolphinscheduler dolphinscheduler-datasource-api + ${project.version} + provided + org.apache.dolphinscheduler dolphinscheduler-storage-api + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided + org.apache.hadoop hadoop-common @@ -132,6 +148,7 @@ + org.apache.hadoop hadoop-client @@ -235,4 +252,21 @@ test + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java index da6cfda89d1d..d3a2cad7eea1 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -99,7 +99,7 @@ private void init() { String defaultFS = hdfsProperties.getDefaultFS(); if (StringUtils.isNotEmpty(defaultFS)) { - configuration.set(Constants.HDFS_DEFAULT_FS, hdfsProperties.getDefaultFS()); + configuration.set(StorageConstants.HDFS_DEFAULT_FS, hdfsProperties.getDefaultFS()); } if (CommonUtils.getKerberosStartupState()) { diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java index ed2aa60615f9..19bbc12cfb32 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import static org.apache.dolphinscheduler.common.constants.Constants.FS_DEFAULT_FS; -import static org.apache.dolphinscheduler.common.constants.Constants.HDFS_ROOT_USER; +import static org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants.FS_DEFAULT_FS; +import static org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants.HDFS_ROOT_USER; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import java.util.Map; @@ -45,7 +45,7 @@ private HdfsStorageProperties getHdfsStorageProperties() { .user(PropertyUtils.getString(HDFS_ROOT_USER)) .defaultFS(PropertyUtils.getString(FS_DEFAULT_FS)) .configurationProperties(configurationProperties) - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) .build(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java index f0bb1543aa52..0038342265b2 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import com.google.auto.service.AutoService; @@ -39,7 +39,7 @@ public StorageOperator createStorageOperate() { private HdfsStorageProperties getHdfsStorageProperties() { return HdfsStorageProperties.builder() .defaultFS(LOCAL_DEFAULT_FS) - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) .build(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java index 8c20d92566e1..0a9b943d5f85 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.dolphinscheduler.spi.enums.ResourceType; import java.nio.file.FileAlreadyExistsException; @@ -46,14 +47,14 @@ class LocalStorageOperatorTest { Paths.get(LocalStorageOperatorTest.class.getResource("/").getFile(), "localStorage").toString(); private static final String tenantCode = "default"; private static final String baseDir = - Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE).toString(); + Paths.get(resourceBaseDir, tenantCode, StorageConstants.RESOURCE_TYPE_FILE).toString(); @SneakyThrows @BeforeEach public void setup() { Files.createDirectories(Paths.get(resourceBaseDir)); - System.clearProperty(Constants.RESOURCE_UPLOAD_PATH); - System.setProperty(Constants.RESOURCE_UPLOAD_PATH, resourceBaseDir); + System.clearProperty(StorageConstants.RESOURCE_UPLOAD_PATH); + System.setProperty(StorageConstants.RESOURCE_UPLOAD_PATH, resourceBaseDir); LocalStorageOperatorFactory localStorageOperatorFactory = new LocalStorageOperatorFactory(); storageOperator = localStorageOperatorFactory.createStorageOperate(); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/pom.xml index d19923498b08..0d22f45a13c8 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/pom.xml @@ -26,21 +26,45 @@ dolphinscheduler-storage-obs + + storage.obs + + - - com.huaweicloud - esdk-obs-java-bundle - org.apache.dolphinscheduler dolphinscheduler-storage-api ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-common ${project.version} + provided + + + + com.huaweicloud + esdk-obs-java-bundle + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java index 6f67ccb36320..1f32cb34afc3 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.plugin.storage.obs; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.commons.lang3.StringUtils; @@ -81,7 +81,7 @@ public void close() throws IOException { public String getStorageBaseDirectory() { // All directory should end with File.separator if (resourceBaseAbsolutePath.startsWith("/")) { - log.warn("{} -> {} should not start with / in obs", Constants.RESOURCE_UPLOAD_PATH, + log.warn("{} -> {} should not start with / in obs", StorageConstants.RESOURCE_UPLOAD_PATH, resourceBaseAbsolutePath); return resourceBaseAbsolutePath.substring(1); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java index 66ba23ee480b..cbc9ee9399b8 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.plugin.storage.obs; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import com.google.auto.service.AutoService; @@ -37,11 +36,11 @@ public StorageOperator createStorageOperate() { private ObsStorageProperties getObsStorageProperties() { return ObsStorageProperties.builder() - .accessKeyId(PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_ID)) - .accessKeySecret(PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_SECRET)) - .bucketName(PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_BUCKET_NAME)) - .endPoint(PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_END_POINT)) - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .accessKeyId(PropertyUtils.getString(StorageConstants.HUAWEI_CLOUD_ACCESS_KEY_ID)) + .accessKeySecret(PropertyUtils.getString(StorageConstants.HUAWEI_CLOUD_ACCESS_KEY_SECRET)) + .bucketName(PropertyUtils.getString(StorageConstants.HUAWEI_CLOUD_OBS_BUCKET_NAME)) + .endPoint(PropertyUtils.getString(StorageConstants.HUAWEI_CLOUD_OBS_END_POINT)) + .resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) .build(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml index 90fa9cb5e0f0..b95492b4da47 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/pom.xml @@ -26,17 +26,45 @@ dolphinscheduler-storage-oss + + storage.oss + + org.apache.dolphinscheduler dolphinscheduler-storage-api ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-common ${project.version} + provided + + + + com.aliyun.oss + aliyun-sdk-oss + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtilsTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssClientFactory.java similarity index 50% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtilsTest.java rename to dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssClientFactory.java index c959121a9a3f..c5652b3deba3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtilsTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssClientFactory.java @@ -15,26 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.api.utils; +package org.apache.dolphinscheduler.plugin.storage.oss; -import java.util.Optional; +import lombok.experimental.UtilityClass; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; -public class JsonPathUtilsTest { +@UtilityClass +public class OssClientFactory { - @Test - public void read() { - String json = "{\"taskInstanceId\":\"123\",\"taskInstanceFinished\":true}"; - Optional optionalBoolean = JsonPathUtils.read(json, "$.taskInstanceFinished"); - Assertions.assertTrue(optionalBoolean.get()); - } - - @Test - public void exist() { - String json = "{\"taskInstanceId\":\"123\",\"taskInstanceFinished\":true}"; - Assertions.assertTrue(JsonPathUtils.exist(json, "$.[?(@.taskInstanceFinished == true)]")); - Assertions.assertFalse(JsonPathUtils.exist(json, "$.[?(@.taskInstanceFinished == false)]")); + public OSS buildOssClient(OssConnection ossConnection) { + return new OSSClientBuilder().build(ossConnection.getEndPoint(), + ossConnection.getAccessKeyId(), ossConnection.getAccessKeySecret()); } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssConnection.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssConnection.java new file mode 100644 index 000000000000..cdeaa9456924 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssConnection.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.oss; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class OssConnection { + + public String accessKeyId; + public String accessKeySecret; + public String endPoint; + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssConstants.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssConstants.java new file mode 100644 index 000000000000..0a059cf39c85 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssConstants.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.oss; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class OssConstants { + + public static final String ALIBABA_CLOUD_ACCESS_KEY_ID = "resource.alibaba.cloud.access.key.id"; + public static final String ALIBABA_CLOUD_ACCESS_KEY_SECRET = "resource.alibaba.cloud.access.key.secret"; + public static final String ALIBABA_CLOUD_REGION = "resource.alibaba.cloud.region"; +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java index afdc6874c9ab..4b142466ced8 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java @@ -17,16 +17,13 @@ package org.apache.dolphinscheduler.plugin.storage.oss; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.factory.OssClientFactory; -import org.apache.dolphinscheduler.common.model.OssConnection; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.commons.lang3.StringUtils; @@ -107,23 +104,23 @@ public void init(OssConnection ossConnection) { } protected String readOssAccessKeyID() { - return PropertyUtils.getString(TaskConstants.ALIBABA_CLOUD_ACCESS_KEY_ID); + return PropertyUtils.getString(OssConstants.ALIBABA_CLOUD_ACCESS_KEY_ID); } protected String readOssAccessKeySecret() { - return PropertyUtils.getString(TaskConstants.ALIBABA_CLOUD_ACCESS_KEY_SECRET); + return PropertyUtils.getString(OssConstants.ALIBABA_CLOUD_ACCESS_KEY_SECRET); } protected String readOssRegion() { - return PropertyUtils.getString(TaskConstants.ALIBABA_CLOUD_REGION); + return PropertyUtils.getString(OssConstants.ALIBABA_CLOUD_REGION); } protected String readOssBucketName() { - return PropertyUtils.getString(Constants.ALIBABA_CLOUD_OSS_BUCKET_NAME); + return PropertyUtils.getString(StorageConstants.ALIBABA_CLOUD_OSS_BUCKET_NAME); } protected String readOssEndPoint() { - return PropertyUtils.getString(Constants.ALIBABA_CLOUD_OSS_END_POINT); + return PropertyUtils.getString(StorageConstants.ALIBABA_CLOUD_OSS_END_POINT); } protected OssConnection buildOssConnection() { @@ -134,7 +131,7 @@ protected OssConnection buildOssConnection() { public String getStorageBaseDirectory() { // All directory should end with File.separator if (resourceBaseAbsolutePath.startsWith("/")) { - log.warn("{} -> {} should not start with / in Oss", Constants.RESOURCE_UPLOAD_PATH, + log.warn("{} -> {} should not start with / in Oss", StorageConstants.RESOURCE_UPLOAD_PATH, resourceBaseAbsolutePath); return resourceBaseAbsolutePath.substring(1); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java index 966cc471aba9..53d65193ee4c 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.storage.oss; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import com.google.auto.service.AutoService; @@ -30,7 +30,8 @@ public class OssStorageOperatorFactory implements StorageOperatorFactory { @Override public StorageOperator createStorageOperate() { - return new OssStorageOperator(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")); + return new OssStorageOperator( + PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml index 2c7a1d2d9969..e0cf69435d0a 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml @@ -26,21 +26,30 @@ dolphinscheduler-storage-s3 + + storage.s3 + + org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-storage-api + ${project.version} + provided org.apache.dolphinscheduler - dolphinscheduler-storage-api + dolphinscheduler-aws-authentication ${project.version} + provided - com.amazonaws - aws-java-sdk-s3 + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + provided @@ -48,6 +57,22 @@ minio test - + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index d70a4c5990d5..2bd479de634a 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.plugin.storage.s3; import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import org.apache.commons.lang3.StringUtils; @@ -75,7 +75,7 @@ public S3StorageOperator(S3StorageProperties s3StorageProperties) { public String getStorageBaseDirectory() { // All directory should end with File.separator if (resourceBaseAbsolutePath.startsWith("/")) { - log.warn("{} -> {} should not start with / in s3", Constants.RESOURCE_UPLOAD_PATH, + log.warn("{} -> {} should not start with / in s3", StorageConstants.RESOURCE_UPLOAD_PATH, resourceBaseAbsolutePath); return resourceBaseAbsolutePath.substring(1); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java index 9b1539a4fbf4..a4073d604570 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.storage.s3; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; import com.google.auto.service.AutoService; @@ -36,9 +36,9 @@ public StorageOperator createStorageOperate() { private S3StorageProperties getS3StorageProperties() { return S3StorageProperties.builder() - .bucketName(PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME)) + .bucketName(PropertyUtils.getString(StorageConstants.AWS_S3_BUCKET_NAME)) .s3Configuration(PropertyUtils.getByPrefix("aws.s3.", "")) - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .resourceUploadPath(PropertyUtils.getString(StorageConstants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) .build(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/pom.xml index 7a61f116c8f4..9919ce0ea85a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/pom.xml @@ -26,43 +26,71 @@ dolphinscheduler-task-aliyunserverlessspark jar + + task.aliyun.serverlessspark + + org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-task-api + ${project.version} provided - - - com.aliyun.oss - aliyun-sdk-oss - - + org.apache.dolphinscheduler - dolphinscheduler-task-api + dolphinscheduler-datasource-all + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api ${project.version} + provided - com.aliyun.oss - aliyun-sdk-oss + org.slf4j + slf4j-reload4j + org.apache.dolphinscheduler - dolphinscheduler-datasource-all + dolphinscheduler-common ${project.version} + provided + com.aliyun emr_serverless_spark20230808 1.0.0 + com.aliyun credentials-java 0.3.0 + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + + + + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml index 7c8f24c71715..144bad0890c9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml @@ -27,47 +27,15 @@ jar - - org.apache.dolphinscheduler - dolphinscheduler-common - org.apache.dolphinscheduler dolphinscheduler-spi - - com.baomidou - mybatis-plus-annotation - - - com.squareup.okhttp3 - okhttp - - - com.jayway.jsonpath - json-path - - - commons-io - commons-io - - - com.google.guava - guava - - - ch.qos.logback - logback-classic - - - org.slf4j - slf4j-api - provided - - org.apache.commons - commons-lang3 + org.apache.dolphinscheduler + dolphinscheduler-common + provided @@ -285,18 +253,12 @@ + io.fabric8 kubernetes-client - - org.apache.commons - commons-collections4 - - - org.projectlombok - lombok - + org.junit.jupiter junit-jupiter diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java index 716947faa2e9..2f538a394510 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractYarnTask.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.api; -import static org.apache.dolphinscheduler.common.constants.Constants.APPID_COLLECT; -import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_COLLECT_WAY; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.APPID_COLLECT; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DEFAULT_COLLECT_WAY; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index 77fd8db7fa5d..008c4a8ba17b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -35,6 +35,9 @@ private TaskConstants() { public static final String FLINK_APPLICATION_REGEX = "JobID \\w+"; + public static final String DATASOURCE_PASSWORD_REGEX = + "(?<=((?i)password((\" : \")|(\":\")|(\\\\\":\\\\\")|(=')))).*?(?=((\")|(\\\\\")|(')))"; + /** * exit code kill */ @@ -99,10 +102,7 @@ private TaskConstants() { * EQUAL SIGN */ public static final String EQUAL_SIGN = "="; - /** - * AT SIGN - */ - public static final String AT_SIGN = "@"; + /** * UNDERLINE */ @@ -304,23 +304,6 @@ private TaskConstants() { */ public static final String D = "-D"; - /** - * datasource encryption salt - */ - public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; - public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; - public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; - - /** - * kerberos - */ - public static final String KERBEROS = "kerberos"; - - /** - * kerberos expire time - */ - public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time"; - /** * java.security.krb5.conf */ @@ -331,38 +314,12 @@ private TaskConstants() { */ public static final String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; - /** - * loginUserFromKeytab user - */ - public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; - - /** - * loginUserFromKeytab path - */ - public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; - - /** - * hadoop.security.authentication - */ - public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; - /** * hadoop.security.authentication */ public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; - /** - * hdfs/s3 configuration - * resource.storage.upload.base.path - */ - public static final String RESOURCE_UPLOAD_PATH = "resource.storage.upload.base.path"; - - /** - * data.quality.jar.dir - */ - public static final String DATA_QUALITY_JAR_DIR = "data-quality.jar.dir"; - public static final String TASK_TYPE_DATA_QUALITY = "DATA_QUALITY"; public static final Set TASK_TYPE_SET_K8S = Sets.newHashSet("K8S", "KUBEFLOW"); @@ -376,19 +333,6 @@ private TaskConstants() { public static final String AZURE_SECRET_TENANT_ID = "resource.azure.tenant.id"; public static final String QUERY_INTERVAL = "resource.query.interval"; - /** - * alibaba cloud config - */ - public static final String ALIBABA_CLOUD_ACCESS_KEY_ID = "resource.alibaba.cloud.access.key.id"; - public static final String ALIBABA_CLOUD_ACCESS_KEY_SECRET = "resource.alibaba.cloud.access.key.secret"; - public static final String ALIBABA_CLOUD_REGION = "resource.alibaba.cloud.region"; - - /** - * huawei cloud config - */ - public static final String HUAWEI_CLOUD_ACCESS_KEY_ID = "resource.huawei.cloud.access.key.id"; - public static final String HUAWEI_CLOUD_ACCESS_KEY_SECRET = "resource.huawei.cloud.access.key.secret"; - /** * use for k8s task */ @@ -428,4 +372,18 @@ private TaskConstants() { String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", GROUP_NAME1, GROUP_NAME2); public static final Pattern SQL_PARAMS_PATTERN = Pattern.compile(SQL_PARAMS_REGEX); + public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; + + public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; + + /** + * fetch applicationId way + */ + public static final String APPID_COLLECT = "appId.collect"; + public static final String DEFAULT_COLLECT_WAY = "log"; + + public static final String WORKFLOW_INSTANCE_ID_MDC_KEY = "workflowInstanceId"; + public static final String TASK_INSTANCE_ID_MDC_KEY = "taskInstanceId"; + + public static final String STAR = "*"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/ApplicationManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/ApplicationManager.java index 0d7761ae6d1c..209f2015f7a0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/ApplicationManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/ApplicationManager.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api.am; -import org.apache.dolphinscheduler.common.enums.ResourceManagerType; +import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceManagerType; public interface ApplicationManager { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java index ac1ce69f7608..a4df2fd19f7a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java @@ -20,10 +20,9 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.SLEEP_TIME_MILLIS; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.UNIQUE_LABEL_NAME; -import org.apache.dolphinscheduler.common.enums.ResourceManagerType; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceManagerType; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.commons.collections4.CollectionUtils; @@ -33,6 +32,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import com.google.auto.service.AutoService; @@ -101,6 +101,7 @@ public ResourceManagerType getResourceManagerType() { * @param kubernetesApplicationManagerContext * @return */ + @SneakyThrows private FilterWatchListDeletable getListenPod(KubernetesApplicationManagerContext kubernetesApplicationManagerContext) { KubernetesClient client = getClient(kubernetesApplicationManagerContext); String labelValue = kubernetesApplicationManagerContext.getLabelValue(); @@ -115,7 +116,7 @@ private FilterWatchListDeletable getListenPod(Kuberne if (!CollectionUtils.isEmpty(podList)) { break; } - ThreadUtils.sleep(SLEEP_TIME_MILLIS); + Thread.sleep(SLEEP_TIME_MILLIS); retryTimes += 1; } @@ -190,6 +191,7 @@ private TaskExecutionStatus getApplicationStatus(KubernetesApplicationManagerCon * @param kubernetesApplicationManagerContext * @return */ + @SneakyThrows public LogWatch getPodLogWatcher(KubernetesApplicationManagerContext kubernetesApplicationManagerContext) { KubernetesClient client = getClient(kubernetesApplicationManagerContext); boolean podIsReady = false; @@ -204,7 +206,7 @@ public LogWatch getPodLogWatcher(KubernetesApplicationManagerContext kubernetesA pod = podList.get(0); String phase = pod.getStatus().getPhase(); if (phase.equals(PENDING) || phase.equals(UNKNOWN)) { - ThreadUtils.sleep(SLEEP_TIME_MILLIS); + Thread.sleep(SLEEP_TIME_MILLIS); } else { podIsReady = true; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/YarnApplicationManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/YarnApplicationManager.java index 9ab19d3e2430..09e5d3c829e8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/YarnApplicationManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/YarnApplicationManager.java @@ -18,9 +18,10 @@ package org.apache.dolphinscheduler.plugin.task.api.am; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResourceManagerType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceManagerType; import java.io.File; import java.nio.charset.StandardCharsets; @@ -96,14 +97,14 @@ private String getKerberosInitCommand() { log.info("get kerberos init command"); StringBuilder kerberosCommandBuilder = new StringBuilder(); boolean hadoopKerberosState = - PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); + PropertyUtils.getBoolean(TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); if (hadoopKerberosState) { kerberosCommandBuilder.append("export KRB5_CONFIG=") - .append(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)) + .append(PropertyUtils.getString(TaskConstants.JAVA_SECURITY_KRB5_CONF_PATH)) .append("\n\n") .append(String.format("kinit -k -t %s %s || true", - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME))) + PropertyUtils.getString(TaskConstants.LOGIN_USER_KEY_TAB_PATH), + PropertyUtils.getString(TaskConstants.LOGIN_USER_KEY_TAB_USERNAME))) .append("\n\n"); log.info("kerberos init command: {}", kerberosCommandBuilder); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceManagerType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ResourceManagerType.java similarity index 93% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceManagerType.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ResourceManagerType.java index 265b0765bf48..ec2c0c4bcb89 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceManagerType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ResourceManagerType.java @@ -15,12 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.enums; +package org.apache.dolphinscheduler.plugin.task.api.enums; public enum ResourceManagerType { - YARN, - KUBERNETES; - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java index 8d3d2513af89..2c8dc1acd2db 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.plugin.task.api.k8s; -import org.apache.dolphinscheduler.common.utils.ClassFilterConstructor; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; +import org.apache.dolphinscheduler.plugin.task.api.utils.ClassFilterConstructor; import org.apache.dolphinscheduler.plugin.task.api.utils.K8sUtils; import java.util.HashMap; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java new file mode 100644 index 000000000000..1c90b03139ef --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.log; + +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.HashSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import ch.qos.logback.classic.pattern.MessageConverter; +import ch.qos.logback.classic.spi.ILoggingEvent; + +import com.google.common.base.Strings; + +/** + * sensitive data log converter + */ +public class SensitiveDataConverter extends MessageConverter { + + private static Pattern multilinePattern; + private static HashSet maskPatterns = + new HashSet<>(Collections.singletonList(TaskConstants.DATASOURCE_PASSWORD_REGEX)); + + @Override + public String convert(ILoggingEvent event) { + + // get original log + String requestLogMsg = event.getFormattedMessage(); + + // desensitization log + return maskSensitiveData(requestLogMsg); + } + + public static void addMaskPattern(String maskPattern) { + maskPatterns.add(maskPattern); + } + + public static String maskSensitiveData(final String logMsg) { + if (StringUtils.isEmpty(logMsg)) { + return logMsg; + } + multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE); + + StringBuffer sb = new StringBuffer(logMsg.length()); + Matcher matcher = multilinePattern.matcher(logMsg); + + while (matcher.find()) { + String password = matcher.group(); + String maskPassword = Strings.repeat(TaskConstants.STAR, password.length()); + matcher.appendReplacement(sb, maskPassword); + } + matcher.appendTail(sb); + + return sb.toString(); + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClassFilterConstructor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ClassFilterConstructor.java similarity index 96% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClassFilterConstructor.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ClassFilterConstructor.java index 467c61c368e4..38bf2b952b06 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClassFilterConstructor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ClassFilterConstructor.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.plugin.task.api.utils; import lombok.extern.slf4j.Slf4j; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java deleted file mode 100644 index 452ed3af8604..000000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.utils; - -import java.util.Optional; -import java.util.Set; - -import lombok.NonNull; -import lombok.experimental.UtilityClass; -import net.minidev.json.JSONArray; - -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.internal.DefaultsImpl; - -@UtilityClass -public class JsonPathUtils { - - static { - Set - @@ -109,18 +108,4 @@ - - - - docker - - - - org.codehaus.mojo - exec-maven-plugin - - - - - diff --git a/dolphinscheduler-tools/src/main/bin/create-demo-processes.sh b/dolphinscheduler-tools/src/main/bin/create-demo-processes.sh old mode 100644 new mode 100755 index f3025994e2e8..329862eb8077 --- a/dolphinscheduler-tools/src/main/bin/create-demo-processes.sh +++ b/dolphinscheduler-tools/src/main/bin/create-demo-processes.sh @@ -17,7 +17,8 @@ # BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd ${BIN_DIR}/../..;pwd)} +TOOLS_HOME=$(cd ${BIN_DIR}/..;pwd) if [ "$DOCKER" != "true" ]; then source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" @@ -26,6 +27,6 @@ fi JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -cp "$TOOLS_HOME/conf":"$TOOLS_HOME/sql":"$TOOLS_HOME/libs/*" \ -Dspring.profiles.active=demo,${DATABASE} \ org.apache.dolphinscheduler.tools.demo.CreateProcessDemo diff --git a/dolphinscheduler-tools/src/main/bin/migrate-lineage.sh b/dolphinscheduler-tools/src/main/bin/migrate-lineage.sh old mode 100644 new mode 100755 index c34ae0e117e5..f33bbd0c6a1d --- a/dolphinscheduler-tools/src/main/bin/migrate-lineage.sh +++ b/dolphinscheduler-tools/src/main/bin/migrate-lineage.sh @@ -17,7 +17,8 @@ # BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd ${BIN_DIR}/../..;pwd)} +TOOLS_HOME=$(cd ${BIN_DIR}/..;pwd) if [ "$DOCKER" != "true" ]; then source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" @@ -26,6 +27,6 @@ fi JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -cp "$TOOLS_HOME/conf":"$TOOLS_HOME/sql":"$TOOLS_HOME/libs/*" \ -Dspring.profiles.active=lineage,${DATABASE} \ org.apache.dolphinscheduler.tools.lineage.MigrateLineage diff --git a/dolphinscheduler-tools/src/main/bin/migrate-resource.sh b/dolphinscheduler-tools/src/main/bin/migrate-resource.sh old mode 100644 new mode 100755 index 223a5d66b35e..b1942cc29b79 --- a/dolphinscheduler-tools/src/main/bin/migrate-resource.sh +++ b/dolphinscheduler-tools/src/main/bin/migrate-resource.sh @@ -17,7 +17,8 @@ # BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd ${BIN_DIR}/../..;pwd)} +TOOLS_HOME=$(cd ${BIN_DIR}/..;pwd) if [ "$DOCKER" != "true" ]; then source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" @@ -26,6 +27,6 @@ fi JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -cp "$TOOLS_HOME/conf":"$TOOLS_HOME/sql":"$TOOLS_HOME/libs/*" \ -Dspring.profiles.active=resource,${DATABASE} \ org.apache.dolphinscheduler.tools.resource.MigrateResource $1 diff --git a/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh index e46b975204f1..31db8af0b555 100755 --- a/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh +++ b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh @@ -17,7 +17,8 @@ # BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd ${BIN_DIR}/../..;pwd)} +TOOLS_HOME=$(cd ${BIN_DIR}/..;pwd) if [ "$DOCKER" != "true" ]; then source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" @@ -26,6 +27,6 @@ fi JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -cp "$TOOLS_HOME/conf":"$TOOLS_HOME/sql":"$TOOLS_HOME/libs/*" \ -Dspring.profiles.active=upgrade,${DATABASE} \ org.apache.dolphinscheduler.tools.datasource.UpgradeDolphinScheduler diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/lineage/MigrateLineage.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/lineage/MigrateLineage.java index 0fab0abaed4e..d5baca112e24 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/lineage/MigrateLineage.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/lineage/MigrateLineage.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.tools.lineage; -import org.apache.dolphinscheduler.tools.resource.MigrateResource; - import java.sql.SQLException; import lombok.extern.slf4j.Slf4j; @@ -37,7 +35,7 @@ public class MigrateLineage { public static void main(String[] args) { - SpringApplication.run(MigrateResource.class, args); + SpringApplication.run(MigrateLineage.class, args); } @Component diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index fd67751ff2fc..5971b4b9ab72 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -227,18 +227,4 @@ - - - - docker - - - - org.codehaus.mojo - exec-maven-plugin - - - - - diff --git a/dolphinscheduler-worker/src/main/bin/start.sh b/dolphinscheduler-worker/src/main/bin/start.sh index c87b51d7a274..0a1b9dd83260 100644 --- a/dolphinscheduler-worker/src/main/bin/start.sh +++ b/dolphinscheduler-worker/src/main/bin/start.sh @@ -18,11 +18,12 @@ set -eo pipefail BIN_DIR=$(dirname $(readlink -f "$0")) -DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd) +DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/../..;pwd) +WORKER_HOME=$(cd ${BIN_DIR}/..;pwd) -source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" +source "$WORKER_HOME/conf/dolphinscheduler_env.sh" -JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh +JVM_ARGS_ENV_FILE=${WORKER_HOME}/bin/jvm_args_env.sh JVM_ARGS="-server" if [ -f $JVM_ARGS_ENV_FILE ]; then @@ -37,12 +38,31 @@ fi JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then - JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" + JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport -DDOCKER=true" fi echo "JAVA_HOME=${JAVA_HOME}" echo "JAVA_OPTS=${JAVA_OPTS}" +MODULES_PATH=( +worker-server +) + +CP="" +for module in ${MODULES_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/$module/libs/*" +done + +PLUGINS_PATH=( +datasource-plugins +storage-plugins +task-plugins +) + +for plugin in ${PLUGINS_PATH[@]}; do + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" +done + $JAVA_HOME/bin/java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ + -cp "$WORKER_HOME/conf""$CP" \ org.apache.dolphinscheduler.server.worker.WorkerServer diff --git a/dolphinscheduler-worker/src/main/resources/banner.txt b/dolphinscheduler-worker/src/main/resources/banner.txt index bf195ee52479..b9b683dc96eb 100644 --- a/dolphinscheduler-worker/src/main/resources/banner.txt +++ b/dolphinscheduler-worker/src/main/resources/banner.txt @@ -8,5 +8,5 @@ ${AnsiColor.BLUE}${AnsiStyle.BOLD} |_| ================================================================================ ${AnsiColor.BLUE}${AnsiStyle.BOLD} -:: DolphinScheduler work server :: ${application.formatted-version} +:: DolphinScheduler worker server :: ${application.formatted-version} ${AnsiStyle.NORMAL} diff --git a/pom.xml b/pom.xml index 48ad5952f83f..f4e553e17be4 100755 --- a/pom.xml +++ b/pom.xml @@ -75,12 +75,11 @@ 2.4 3.0.0-M6 3.1.1 - 3.2.1 + 3.3.0 2.2.0 1.14.0 2.27.2 0.8.8 - false 2.7 1.0.1 false @@ -455,6 +454,37 @@ org.apache.maven.plugins maven-shade-plugin ${maven-shade-plugin.version} + + true + shade + false + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + shade + + package + + + + + + + + org.apache.maven.plugins @@ -465,64 +495,6 @@ org.codehaus.mojo exec-maven-plugin ${exec-maven-plugin.version} - - - docker-build - - exec - - package - - ${docker.build.skip} - - 1 - - docker - ${project.basedir} - - buildx - build - --load - --no-cache - -t - ${docker.hub}/${docker.repo}:${docker.tag} - -t - ${docker.hub}/${docker.repo}:latest - ${project.basedir} - --file=src/main/docker/Dockerfile - - - - - docker-push - - exec - - deploy - - ${docker.push.skip} - - 1 - - docker - ${project.basedir} - - buildx - build - --platform - linux/amd64,linux/arm64 - --no-cache - --push - -t - ${docker.hub}/${docker.repo}:${docker.tag} - -t - ${docker.hub}/${docker.repo}:latest - ${project.basedir} - --file=src/main/docker/Dockerfile - - - - org.owasp diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh index 76a6f92c17ec..eaeb6fe3f3ab 100755 --- a/tools/dependencies/check-LICENSE.sh +++ b/tools/dependencies/check-LICENSE.sh @@ -28,6 +28,8 @@ tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --stri # licenses echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt +echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}-shade.jar' exec:exec | tee -a self-modules.txt + echo '=== Distributed dependencies: ' && find dist -name "*.jar" -exec basename {} \; | sort | uniq | tee all-dependencies.txt # Exclude all self modules(jars) to generate all third-party dependencies diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index c6780804d4f1..e9d3e529ffa7 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -63,7 +63,6 @@ cron-utils-9.1.6.jar curator-client-5.3.0.jar curator-framework-5.3.0.jar curator-recipes-5.3.0.jar -curator-test-5.3.0.jar curvesapi-1.06.jar datanucleus-api-jdo-4.2.4.jar datanucleus-core-4.1.17.jar @@ -165,7 +164,6 @@ jpam-1.1.jar jsch-0.1.55.jar json-1.8.jar json-20140107.jar -json-path-2.7.0.jar json-smart-2.4.8.jar json-utils-2.17.282.jar jsp-api-2.1.jar @@ -434,9 +432,9 @@ azure-core-management-1.10.1.jar api-common-2.6.0.jar auto-value-1.10.1.jar auto-value-annotations-1.10.4.jar +conscrypt-openjdk-uber-2.5.2.jar bcpkix-jdk15on-1.67.jar bcprov-jdk15on-1.67.jar -conscrypt-openjdk-uber-2.5.2.jar gapic-google-cloud-storage-v2-2.18.0-alpha.jar gax-2.23.0.jar gax-grpc-2.23.0.jar