diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index 5be3ba27ff9f..09906af1e394 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -507,6 +507,14 @@ export default { title: 'Kubernetes Deployment', link: '/en-us/docs/dev/user_doc/guide/installation/kubernetes.html', }, + { + title: 'General Setting', + link: '/en-us/docs/dev/user_doc/guide/howto/general-setting.html', + }, + { + title: 'Datasource Setting', + link: '/en-us/docs/dev/user_doc/guide/howto/datasource-setting.html', + }, { title: 'integration', children: [ @@ -713,23 +721,6 @@ export default { }, ], }, - { - title: 'FAQ', - children: [ - { - title: 'General Setting', - link: '/en-us/docs/dev/user_doc/guide/howto/general-setting.html', - }, - { - title: 'Datasource Setting', - link: '/en-us/docs/dev/user_doc/guide/howto/datasource-setting.html', - }, - { - title: 'Others', - link: '/en-us/docs/release/faq.html', - } - ], - }, { title: 'Older Versions', children: [ @@ -1215,6 +1206,14 @@ export default { title: 'Kubernetes部署(Kubernetes)', link: '/zh-cn/docs/dev/user_doc/guide/installation/kubernetes.html', }, + { + title: '通用设置', + link: '/zh-cn/docs/dev/user_doc/guide/howto/general-setting.html', + }, + { + title: '数据源设置', + link: '/zh-cn/docs/dev/user_doc/guide/howto/datasource-setting.html', + }, { title: '集成', children: [ @@ -1421,23 +1420,6 @@ export default { }, ], }, - { - title: 'FAQ', - children: [ - { - title: '通用设置', - link: '/zh-cn/docs/dev/user_doc/guide/howto/general-setting.html', - }, - { - title: '数据源设置', - link: '/zh-cn/docs/dev/user_doc/guide/howto/datasource-setting.html', - }, - { - title: '其他', - link: '/zh-cn/docs/release/faq.html', - }, - ], - }, { title: '历史版本', children: [ diff --git a/docs/configs/index.md.jsx b/docs/configs/index.md.jsx index e3d664bd8fe0..ced4974d5079 100644 --- a/docs/configs/index.md.jsx +++ b/docs/configs/index.md.jsx @@ -62,7 +62,7 @@ import docs317Config from '../../../site_config/docs3-1-7'; import docs318Config from '../../../site_config/docs3-1-8'; import docs319Config from '../../../site_config/docs3-1-9'; import docs320Config from '../../../site_config/docs3-2-0'; -import docs310Config from '../../../site_config/docs3-2-1'; +import docs321Config from '../../../site_config/docs3-2-1'; import docs322Config from '../../../site_config/docs3-2-2'; import docsDevConfig from '../../../site_config/docsdev'; diff --git a/docs/docs/en/guide/howto/datasource-setting.md b/docs/docs/en/guide/installation/datasource-setting.md similarity index 97% rename from docs/docs/en/guide/howto/datasource-setting.md rename to docs/docs/en/guide/installation/datasource-setting.md index 7cc89cafb67d..106dfd77f4a5 100644 --- a/docs/docs/en/guide/howto/datasource-setting.md +++ b/docs/docs/en/guide/installation/datasource-setting.md @@ -7,7 +7,7 @@ We here use MySQL as an example to illustrate how to configure an external datab > NOTE: If you use MySQL, you need to manually download [mysql-connector-java driver][mysql] (8.0.16) and move it to the libs directory of DolphinScheduler > which is `api-server/libs` and `alert-server/libs` and `master-server/libs` and `worker-server/libs`. -* First of all, follow the instructions in [datasource-setting](datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database +* First of all, follow the instructions in `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database * Set the following environment variables in your terminal with your database address, username and password for `{address}`, `{user}` and `{password}`: ```shell diff --git a/docs/docs/en/guide/howto/general-setting.md b/docs/docs/en/guide/installation/general-setting.md similarity index 100% rename from docs/docs/en/guide/howto/general-setting.md rename to docs/docs/en/guide/installation/general-setting.md diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index 851b0501a1c7..57550482968a 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -123,7 +123,7 @@ export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HI ## Initialize the Database -Follow the instructions in [datasource-setting](../howto/datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database +Follow the instructions in [datasource-setting](datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database ## Start DolphinScheduler diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index 646c1d48b0ab..74b276df70ce 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -85,7 +85,7 @@ bash ./bin/dolphinscheduler-daemon.sh status standalone-server ## Database Configuration Standalone server use H2 database as its metadata store, it is easy and users do not need to start database before they set up server. -But if user want to store metabase in other database like MySQL or PostgreSQL, they have to change some configuration. Follow the instructions in [datasource-setting](../howto/datasource-setting.md) `Standalone Switching Metadata Database Configuration` section to create and initialize database +But if user want to store metabase in other database like MySQL or PostgreSQL, they have to change some configuration. Follow the instructions in [datasource-setting](datasource-setting.md) `Standalone Switching Metadata Database Configuration` section to create and initialize database > Note: DS uses the /tmp/dolphinscheduler directory as the resource center by default. If you need to change the directory of the resource center, change the resource items in the conf/common.properties file diff --git a/docs/docs/zh/guide/howto/datasource-setting.md b/docs/docs/zh/guide/installation/datasource-setting.md similarity index 97% rename from docs/docs/zh/guide/howto/datasource-setting.md rename to docs/docs/zh/guide/installation/datasource-setting.md index 63e9e359eb7f..36d2d7ba18f6 100644 --- a/docs/docs/zh/guide/howto/datasource-setting.md +++ b/docs/docs/zh/guide/installation/datasource-setting.md @@ -6,7 +6,7 @@ > 如果使用 MySQL 需要手动下载 [mysql-connector-java 驱动][mysql] (8.0.16) 并移动到 DolphinScheduler 的每个模块的 libs 目录下，其中包括 `api-server/libs` 和 `alert-server/libs` 和 `master-server/libs` 和 `worker-server/libs`。 -* 首先，参照 [数据源配置](datasource-setting.md) `伪分布式/分布式安装初始化数据库` 创建并初始化数据库 +* 首先，参照 `伪分布式/分布式安装初始化数据库` 创建并初始化数据库 * 在你的命令行设定下列环境变量，将 `{address}`, `{user}` 和 `{password}` 改为你数据库的地址, 用户名和密码 ```shell diff --git a/docs/docs/zh/guide/howto/general-setting.md b/docs/docs/zh/guide/installation/general-setting.md similarity index 100% rename from docs/docs/zh/guide/howto/general-setting.md rename to docs/docs/zh/guide/installation/general-setting.md diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index 0449951d4d97..eb1a1502af98 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -121,7 +121,7 @@ export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HI ## 初始化数据库 -请参考 [数据源配置] `伪分布式/分布式安装初始化数据库` 创建并初始化数据库 +请参考 [数据源配置](datasource-setting.md) `伪分布式/分布式安装初始化数据库` 创建并初始化数据库 ## 启动 DolphinScheduler diff --git a/docs/docs/zh/guide/installation/standalone.md b/docs/docs/zh/guide/installation/standalone.md index bc280d9ed746..e64c8f8b5357 100644 --- a/docs/docs/zh/guide/installation/standalone.md +++ b/docs/docs/zh/guide/installation/standalone.md @@ -83,7 +83,7 @@ bash ./bin/dolphinscheduler-daemon.sh status standalone-server ## 配置数据库 Standalone server 使用 H2 数据库作为其元数据存储数据，这是为了上手简单，用户在启动服务器之前不需要启动数据库。但是如果用户想将元数据库存储在 -MySQL 或 PostgreSQL 等其他数据库中，必须更改一些配置。请参考 [数据源配置](../howto/datasource-setting.md) `Standalone 切换元数据库` 创建并初始化数据库 +MySQL 或 PostgreSQL 等其他数据库中，必须更改一些配置。请参考 [数据源配置](datasource-setting.md) `Standalone 切换元数据库` 创建并初始化数据库 > **_注意_**: DS默认使用本地模式的目录 /tmp/dolphinscheduler 作为资源中心, 如果需要修改资源中心目录, 请修改配置文件 conf/common.properties 中 resource 的相关配置项 diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java index 1676df7e01c1..6ee509a1dba8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java @@ -40,9 +40,38 @@ public class SwitchTaskUtils { private static final NashornSandbox sandbox; private static final String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*"; + public static final String NASHORN_POLYFILL_ARRAY_PROTOTYPE_INCLUDES = + "if (!Array.prototype.includes) {" + + " Object.defineProperty(Array.prototype, 'includes', {" + + " value: function(valueToFind, fromIndex) {" + + " if (this == null) {" + + " throw new TypeError('\"this\" is null or not defined');" + + " }" + + " var o = Object(this);" + + " var len = o.length >>> 0;" + + " if (len === 0) { return false; }" + + " var n = fromIndex | 0;" + + " var k = Math.max(n >= 0 ? n : len - Math.abs(n), 0);" + + " function sameValueZero(x, y) {" + + " return x === y || (typeof x === 'number' && " + + " typeof y === 'number' && isNaN(x) && isNaN(y));" + + " }" + + " while (k < len) {" + + " if (sameValueZero(o[k], valueToFind)) { return true; }" + + " k++;" + + " }" + + " return false;" + + " }" + + " });" + + "}"; static { sandbox = NashornSandboxes.create(); + try { + sandbox.eval(NASHORN_POLYFILL_ARRAY_PROTOTYPE_INCLUDES); + } catch (ScriptException e) { + log.error("failed to load Nashorn polyfill", e); + } } public static boolean evaluate(String expression) throws ScriptException { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java index 34785ada4757..81b8b1bc9c99 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java @@ -71,4 +71,16 @@ public void testIllegalCondition() { }); } + + @Test + public void testIncludes() throws ScriptException { + String content = "['abc','def'].includes('abc')"; + boolean result = SwitchTaskUtils.evaluate(content); + Assertions.assertTrue(result); + + SwitchTaskUtils.evaluate(SwitchTaskUtils.NASHORN_POLYFILL_ARRAY_PROTOTYPE_INCLUDES); + result = SwitchTaskUtils.evaluate(content); + Assertions.assertTrue(result); + } + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java index e8ca6370f009..f48169c5e150 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java @@ -319,6 +319,8 @@ public TaskResponse run(String k8sParameterStr) throws Exception { Thread.currentThread().interrupt(); result.setExitStatusCode(EXIT_CODE_FAILURE); throw e; + } finally { + ProcessUtils.removeK8sClientCache(taskRequest.getTaskAppId()); } return result; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java index 555c69bc5c85..27105184b675 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java @@ -243,4 +243,10 @@ public static LogWatch getPodLogWatcher(K8sTaskExecutionContext k8sTaskExecution .getPodLogWatcher( new KubernetesApplicationManagerContext(k8sTaskExecutionContext, taskAppId, containerName)); } + + public static void removeK8sClientCache(String taskAppId) { + KubernetesApplicationManager applicationManager = + (KubernetesApplicationManager) applicationManagerMap.get(ResourceManagerType.KUBERNETES); + applicationManager.removeCache(taskAppId); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java index 14caf470d7aa..53bd77dc2091 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java @@ -84,4 +84,15 @@ public void testGetPidsStr() throws Exception { Assertions.assertEquals(exceptPidsStr3, actualPidsStr3); } + @Test + public void tetRemoveK8sClientCache() { + Assertions.assertDoesNotThrow(() -> { + ProcessUtils.removeK8sClientCache("a"); + }); + + Assertions.assertThrows(Exception.class, () -> { + ProcessUtils.removeK8sClientCache(null); + }); + } + }