Merge branch 'apache:dev' into dev

docs/configs/docsdev.js

@@ -507,6 +507,14 @@ export default {
                         title: 'Kubernetes Deployment',
                         link: '/en-us/docs/dev/user_doc/guide/installation/kubernetes.html',
                     },
+                    {
+                        title: 'General Setting',
+                        link: '/en-us/docs/dev/user_doc/guide/howto/general-setting.html',
+                    },
+                    {
+                        title: 'Datasource Setting',
+                        link: '/en-us/docs/dev/user_doc/guide/howto/datasource-setting.html',
+                    },
                     {
                         title: 'integration',
                         children: [
@@ -713,23 +721,6 @@ export default {
                     },
                 ],
             },
-            {
-                title: 'FAQ',
-                children: [
-                    {
-                        title: 'General Setting',
-                        link: '/en-us/docs/dev/user_doc/guide/howto/general-setting.html',
-                    },
-                    {
-                        title: 'Datasource Setting',
-                        link: '/en-us/docs/dev/user_doc/guide/howto/datasource-setting.html',
-                    },
-                    {
-                        title: 'Others',
-                        link: '/en-us/docs/release/faq.html',
-                    }
-                ],
-            },
             {
                 title: 'Older Versions',
                 children: [
@@ -1215,6 +1206,14 @@ export default {
                         title: 'Kubernetes部署(Kubernetes)',
                         link: '/zh-cn/docs/dev/user_doc/guide/installation/kubernetes.html',
                     },
+                    {
+                        title: '通用设置',
+                        link: '/zh-cn/docs/dev/user_doc/guide/howto/general-setting.html',
+                    },
+                    {
+                        title: '数据源设置',
+                        link: '/zh-cn/docs/dev/user_doc/guide/howto/datasource-setting.html',
+                    },
                     {
                         title: '集成',
                         children: [
@@ -1421,23 +1420,6 @@ export default {
                     },
                 ],
             },
-            {
-                title: 'FAQ',
-                children: [
-                    {
-                        title: '通用设置',
-                        link: '/zh-cn/docs/dev/user_doc/guide/howto/general-setting.html',
-                    },
-                    {
-                        title: '数据源设置',
-                        link: '/zh-cn/docs/dev/user_doc/guide/howto/datasource-setting.html',
-                    },
-                    {
-                        title: '其他',
-                        link: '/zh-cn/docs/release/faq.html',
-                    },
-                ],
-            },
             {
                 title: '历史版本',
                 children: [

docs/configs/index.md.jsx

@@ -62,7 +62,7 @@ import docs317Config from '../../../site_config/docs3-1-7';
 import docs318Config from '../../../site_config/docs3-1-8';
 import docs319Config from '../../../site_config/docs3-1-9';
 import docs320Config from '../../../site_config/docs3-2-0';
-import docs310Config from '../../../site_config/docs3-2-1';
+import docs321Config from '../../../site_config/docs3-2-1';
 import docs322Config from '../../../site_config/docs3-2-2';
 import docsDevConfig from '../../../site_config/docsdev';
 

docs/docs/en/guide/howto/datasource-setting.md → docs/docs/en/guide/installation/datasource-setting.md

@@ -7,7 +7,7 @@ We here use MySQL as an example to illustrate how to configure an external datab
 > NOTE: If you use MySQL, you need to manually download [mysql-connector-java driver][mysql] (8.0.16) and move it to the libs directory of DolphinScheduler
 > which is `api-server/libs` and `alert-server/libs` and `master-server/libs` and `worker-server/libs`.
 
-* First of all, follow the instructions in [datasource-setting](datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database
+* First of all, follow the instructions in `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database
 * Set the following environment variables in your terminal with your database address, username and password for `{address}`, `{user}` and `{password}`:
 
 ```shell

docs/docs/en/guide/installation/pseudo-cluster.md

@@ -123,7 +123,7 @@ export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HI
 
 ## Initialize the Database
 
-Follow the instructions in [datasource-setting](../howto/datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database
+Follow the instructions in [datasource-setting](datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database
 
 ## Start DolphinScheduler
 

docs/docs/en/guide/installation/standalone.md

@@ -85,7 +85,7 @@ bash ./bin/dolphinscheduler-daemon.sh status standalone-server
 ## Database Configuration
 
 Standalone server use H2 database as its metadata store, it is easy and users do not need to start database before they set up server.
-But if user want to store metabase in other database like MySQL or PostgreSQL, they have to change some configuration. Follow the instructions in [datasource-setting](../howto/datasource-setting.md) `Standalone Switching Metadata Database Configuration` section to create and initialize database
+But if user want to store metabase in other database like MySQL or PostgreSQL, they have to change some configuration. Follow the instructions in [datasource-setting](datasource-setting.md) `Standalone Switching Metadata Database Configuration` section to create and initialize database
 
 > Note: DS uses the /tmp/dolphinscheduler directory as the resource center by default. If you need to change the directory of the resource center, change the resource items in the conf/common.properties file
 

docs/docs/zh/guide/howto/datasource-setting.md → docs/docs/zh/guide/installation/datasource-setting.md

@@ -6,7 +6,7 @@
 
 > 如果使用 MySQL 需要手动下载 [mysql-connector-java 驱动][mysql] (8.0.16) 并移动到 DolphinScheduler 的每个模块的 libs 目录下，其中包括 `api-server/libs` 和 `alert-server/libs` 和 `master-server/libs` 和 `worker-server/libs`。
 
-* 首先，参照 [数据源配置](datasource-setting.md) `伪分布式/分布式安装初始化数据库` 创建并初始化数据库
+* 首先，参照 `伪分布式/分布式安装初始化数据库` 创建并初始化数据库
 * 在你的命令行设定下列环境变量，将 `{address}`, `{user}` 和 `{password}` 改为你数据库的地址, 用户名和密码
 
 ```shell

docs/docs/zh/guide/installation/pseudo-cluster.md

@@ -121,7 +121,7 @@ export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HI
 
 ## 初始化数据库
 
-请参考 [数据源配置] `伪分布式/分布式安装初始化数据库` 创建并初始化数据库
+请参考 [数据源配置](datasource-setting.md) `伪分布式/分布式安装初始化数据库` 创建并初始化数据库
 
 ## 启动 DolphinScheduler
 

docs/docs/zh/guide/installation/standalone.md

@@ -83,7 +83,7 @@ bash ./bin/dolphinscheduler-daemon.sh status standalone-server
 ## 配置数据库
 
 Standalone server 使用 H2 数据库作为其元数据存储数据，这是为了上手简单，用户在启动服务器之前不需要启动数据库。但是如果用户想将元数据库存储在
-MySQL 或 PostgreSQL 等其他数据库中，必须更改一些配置。请参考 [数据源配置](../howto/datasource-setting.md) `Standalone 切换元数据库` 创建并初始化数据库
+MySQL 或 PostgreSQL 等其他数据库中，必须更改一些配置。请参考 [数据源配置](datasource-setting.md) `Standalone 切换元数据库` 创建并初始化数据库
 
 > **_注意_**: DS默认使用本地模式的目录 /tmp/dolphinscheduler 作为资源中心, 如果需要修改资源中心目录, 请修改配置文件 conf/common.properties 中 resource 的相关配置项
 

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java

@@ -40,9 +40,38 @@ public class SwitchTaskUtils {
 
     private static final NashornSandbox sandbox;
     private static final String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*";
+    public static final String NASHORN_POLYFILL_ARRAY_PROTOTYPE_INCLUDES =
+            "if (!Array.prototype.includes) {" +
+                    "   Object.defineProperty(Array.prototype, 'includes', {" +
+                    "       value: function(valueToFind, fromIndex) {" +
+                    "           if (this == null) {" +
+                    "               throw new TypeError('\"this\" is null or not defined');" +
+                    "           }" +
+                    "           var o = Object(this);" +
+                    "           var len = o.length >>> 0;" +
+                    "           if (len === 0) { return false; }" +
+                    "           var n = fromIndex | 0;" +
+                    "           var k = Math.max(n >= 0 ? n : len - Math.abs(n), 0);" +
+                    "           function sameValueZero(x, y) {" +
+                    "               return x === y || (typeof x === 'number' && " +
+                    "                   typeof y === 'number' && isNaN(x) && isNaN(y));" +
+                    "           }" +
+                    "           while (k < len) {" +
+                    "               if (sameValueZero(o[k], valueToFind)) { return true; }" +
+                    "               k++;" +
+                    "           }" +
+                    "           return false;" +
+                    "       }" +
+                    "   });" +
+                    "}";
 
     static {
         sandbox = NashornSandboxes.create();
+        try {
+            sandbox.eval(NASHORN_POLYFILL_ARRAY_PROTOTYPE_INCLUDES);
+        } catch (ScriptException e) {
+            log.error("failed to load Nashorn polyfill", e);
+        }
     }
 
     public static boolean evaluate(String expression) throws ScriptException {

dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java

@@ -71,4 +71,16 @@ public void testIllegalCondition() {
         });
 
     }
+
+    @Test
+    public void testIncludes() throws ScriptException {
+        String content = "['abc','def'].includes('abc')";
+        boolean result = SwitchTaskUtils.evaluate(content);
+        Assertions.assertTrue(result);
+
+        SwitchTaskUtils.evaluate(SwitchTaskUtils.NASHORN_POLYFILL_ARRAY_PROTOTYPE_INCLUDES);
+        result = SwitchTaskUtils.evaluate(content);
+        Assertions.assertTrue(result);
+    }
+
 }

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java

@@ -319,6 +319,8 @@ public TaskResponse run(String k8sParameterStr) throws Exception {
             Thread.currentThread().interrupt();
             result.setExitStatusCode(EXIT_CODE_FAILURE);
             throw e;
+        } finally {
+            ProcessUtils.removeK8sClientCache(taskRequest.getTaskAppId());
         }
         return result;
     }

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java

@@ -243,4 +243,10 @@ public static LogWatch getPodLogWatcher(K8sTaskExecutionContext k8sTaskExecution
                 .getPodLogWatcher(
                         new KubernetesApplicationManagerContext(k8sTaskExecutionContext, taskAppId, containerName));
     }
+
+    public static void removeK8sClientCache(String taskAppId) {
+        KubernetesApplicationManager applicationManager =
+                (KubernetesApplicationManager) applicationManagerMap.get(ResourceManagerType.KUBERNETES);
+        applicationManager.removeCache(taskAppId);
+    }
 }

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java

@@ -84,4 +84,15 @@ public void testGetPidsStr() throws Exception {
         Assertions.assertEquals(exceptPidsStr3, actualPidsStr3);
     }
 
+    @Test
+    public void tetRemoveK8sClientCache() {
+        Assertions.assertDoesNotThrow(() -> {
+            ProcessUtils.removeK8sClientCache("a");
+        });
+
+        Assertions.assertThrows(Exception.class, () -> {
+            ProcessUtils.removeK8sClientCache(null);
+        });
+    }
+
 }