From 42894cffe8baa5cba4b65ce7a530a79fe39b9530 Mon Sep 17 00:00:00 2001 From: chenghongchen Date: Tue, 22 Oct 2024 15:40:50 +0800 Subject: [PATCH 1/3] =?UTF-8?q?[fix][service]=20=E9=80=9A=E8=BF=87?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2dfs.hosts=E9=85=8D=E7=BD=AE=E7=9A=84=E5=80=BC?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E7=99=BD=E5=90=8D=E5=8D=95=E6=96=87=E4=BB=B6?= =?UTF-8?q?=20(#622)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasophon/api/utils/ProcessUtils.java | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/datasophon-service/src/main/java/com/datasophon/api/utils/ProcessUtils.java b/datasophon-service/src/main/java/com/datasophon/api/utils/ProcessUtils.java index 8c383dff..4381fcfa 100644 --- a/datasophon-service/src/main/java/com/datasophon/api/utils/ProcessUtils.java +++ b/datasophon-service/src/main/java/com/datasophon/api/utils/ProcessUtils.java @@ -143,9 +143,7 @@ public static void saveServiceInstallInfo(ServiceRoleInfo serviceRoleInfo) { SpringTool.getApplicationContext().getBean(ClusterServiceRoleInstanceWebuisService.class); ClusterServiceInstanceRoleGroupService roleGroupService = SpringTool.getApplicationContext().getBean(ClusterServiceInstanceRoleGroupService.class); - ClusterInfoEntity clusterInfo = clusterInfoService.getById(serviceRoleInfo.getClusterId()); - ClusterServiceInstanceEntity clusterServiceInstance = serviceInstanceService.getServiceInstanceByClusterIdAndServiceName(serviceRoleInfo.getClusterId(), serviceRoleInfo.getParentName()); @@ -177,7 +175,6 @@ public static void saveServiceInstallInfo(ServiceRoleInfo serviceRoleInfo) { } Integer roleGroupId = (Integer) CacheUtils.get("UseRoleGroup_" + clusterServiceInstance.getId()); ClusterServiceInstanceRoleGroup roleGroup = roleGroupService.getById(roleGroupId); - // save role instance ClusterServiceRoleInstanceEntity roleInstanceEntity = serviceRoleInstanceService .getOneServiceRole(serviceRoleInfo.getName(), serviceRoleInfo.getHostname(), clusterInfo.getId()); @@ -222,20 +219,15 @@ public static void saveServiceInstallInfo(ServiceRoleInfo serviceRoleInfo) { webuisService.save(webuis); globalVariables.remove("${host}"); } - } } - } - public static void saveHostInstallInfo(StartWorkerMessage message, String clusterCode, ClusterHostService clusterHostService) { ClusterInfoService clusterInfoService = SpringTool.getApplicationContext().getBean(ClusterInfoService.class); ClusterHostDO clusterHostDO = new ClusterHostDO(); BeanUtil.copyProperties(message, clusterHostDO); - ClusterInfoEntity cluster = clusterInfoService.getClusterByClusterCode(clusterCode); - clusterHostDO.setClusterId(cluster.getId()); clusterHostDO.setCheckTime(new Date()); clusterHostDO.setRack("/default-rack"); @@ -246,7 +238,6 @@ public static void saveHostInstallInfo(StartWorkerMessage message, String cluste clusterHostDO.setManaged(MANAGED.YES); clusterHostService.save(clusterHostDO); } - public static void updateCommandStateToFailed(List commandIds) { for (String commandId : commandIds) { logger.info("command id is {}", commandId); @@ -283,12 +274,10 @@ public static void updateCommandStateToFailed(List commandIds) { } } } - public static void tellCommandActorResult(String serviceName, ExecuteServiceRoleCommand executeServiceRoleCommand, ServiceExecuteState state) { ActorRef serviceExecuteResultActor = ActorUtils.getLocalActor(ServiceExecuteResultActor.class, ActorUtils.getActorRefName(ServiceExecuteResultActor.class)); - ServiceExecuteResultMessage serviceExecuteResultMessage = new ServiceExecuteResultMessage(); serviceExecuteResultMessage.setServiceExecuteState(state); serviceExecuteResultMessage.setDag(executeServiceRoleCommand.getDag()); @@ -302,15 +291,12 @@ public static void tellCommandActorResult(String serviceName, ExecuteServiceRole serviceExecuteResultMessage.setErrorTaskList(executeServiceRoleCommand.getErrorTaskList()); serviceExecuteResultMessage.setReadyToSubmitTaskList(executeServiceRoleCommand.getReadyToSubmitTaskList()); serviceExecuteResultMessage.setCompleteTaskList(executeServiceRoleCommand.getCompleteTaskList()); - serviceExecuteResultActor.tell(serviceExecuteResultMessage, ActorRef.noSender()); } - public static ClusterServiceCommandHostCommandEntity handleCommandResult(String hostCommandId, Boolean execResult, String execOut) { ClusterServiceCommandHostCommandService service = SpringTool.getApplicationContext().getBean(ClusterServiceCommandHostCommandService.class); - ClusterServiceCommandHostCommandEntity hostCommand = service.getByHostCommandId(hostCommandId); hostCommand.setCommandProgress(100); if (execResult) { @@ -335,14 +321,12 @@ public static ClusterServiceCommandHostCommandEntity handleCommandResult(String } else { message.setServiceRoleType(ServiceRoleType.WORKER); } - ActorRef commandActor = ActorUtils.getLocalActor(ServiceCommandActor.class, "commandActor"); ActorUtils.actorSystem.scheduler().scheduleOnce(FiniteDuration.apply( 1L, TimeUnit.SECONDS), commandActor, message, ActorUtils.actorSystem.dispatcher(), ActorRef.noSender()); - return hostCommand; } @@ -469,17 +453,36 @@ public static void hdfsEcMethond(Integer serviceInstanceId, ClusterServiceRoleIn .eq(ClusterServiceRoleInstanceEntity::getServiceId, serviceInstanceId) .eq(ClusterServiceRoleInstanceEntity::getServiceRoleName, roleName) .list(); - - // 更新namenode节点的whitelist白名单 + ClusterVariableService variableService = + SpringTool.getApplicationContext().getBean(ClusterVariableService.class); + // 更新namenode节点的whitelist白名单或blacklist黑名单，分别是dfs.hosts、dfs.hosts.exclude 对应的配置文件 for (ClusterServiceRoleInstanceEntity namenode : namenodes) { ActorSelection actorSelection = ActorUtils.actorSystem.actorSelection( "akka.tcp://datasophon@" + namenode.getHostname() + ":2552/user/worker/fileOperateActor"); ActorSelection execCmdActor = ActorUtils.actorSystem.actorSelection( "akka.tcp://datasophon@" + namenode.getHostname() + ":2552/user/worker/executeCmdActor"); Timeout timeout = new Timeout(Duration.create(180, TimeUnit.SECONDS)); + // DataNode 节点的 hostname 作为文件内容 FileOperateCommand fileOperateCommand = new FileOperateCommand(); fileOperateCommand.setLines(list); - fileOperateCommand.setPath(Constants.INSTALL_PATH + "/hadoop/etc/hadoop/" + type); + // fileOperateCommand.setPath(Constants.INSTALL_PATH + "/hadoop/etc/hadoop/" + type); + if ("whitelist".equals(type)) { + // 优先获取通过页面白名单文件变量配置的路径，而不是写死文件路径 + ClusterVariable clusterVariable = variableService.getVariableByVariableName("${dfs.hosts}", namenode.getClusterId()); + if (!Objects.isNull(clusterVariable) && !StringUtils.isBlank(clusterVariable.getVariableValue())) { + logger.info("dfs.hosts value is {}", clusterVariable.getVariableValue()); + fileOperateCommand.setPath(clusterVariable.getVariableValue()); + } else { + fileOperateCommand.setPath(Constants.INSTALL_PATH + "/hadoop/etc/hadoop/" + type); + } + } else { + ClusterVariable clusterVariable = variableService.getVariableByVariableName("${dfs.hosts.exclude}", namenode.getClusterId()); + if (!Objects.isNull(clusterVariable) && !StringUtils.isBlank(clusterVariable.getVariableValue())) { + fileOperateCommand.setPath(clusterVariable.getVariableValue()); + } else { + fileOperateCommand.setPath(Constants.INSTALL_PATH + "/hadoop/etc/hadoop/" + type); + } + } Future future = Patterns.ask(actorSelection, fileOperateCommand, timeout); ExecResult fileOperateResult = (ExecResult) Await.result(future, timeout.duration()); if (Objects.nonNull(fileOperateResult) && fileOperateResult.getExecResult()) { From 970ce76b831f1cf82d6d1e1dad3394f4e5d7787f Mon Sep 17 00:00:00 2001 From: chenghongchen Date: Tue, 22 Oct 2024 15:50:19 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=94=AF=E6=8C=81=20value=20=E4=B8=BA?= =?UTF-8?q?=E5=A4=9A=E8=A1=8C=E9=85=8D=E7=BD=AE=EF=BC=8C=E5=A6=82=20dfs.ha?= =?UTF-8?q?.fencing.methods=20=E5=90=8C=E6=97=B6=E9=85=8D=E7=BD=AE=20sshfe?= =?UTF-8?q?nce=20=E5=92=8C=20shell(/bin/true)=EF=BC=8C=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=80=BC=E4=B8=BAsshfence\nshell(/bin/true)?= =?UTF-8?q?=E5=8D=B3=E5=8F=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datasophon-worker/src/main/resources/templates/xml.ftl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datasophon-worker/src/main/resources/templates/xml.ftl b/datasophon-worker/src/main/resources/templates/xml.ftl index 91773ace..438432d0 100644 --- a/datasophon-worker/src/main/resources/templates/xml.ftl +++ b/datasophon-worker/src/main/resources/templates/xml.ftl @@ -4,7 +4,15 @@ <#list itemList as item> ${item.name} +<#if item.value?contains("\\n")> + +<#list item.value?split("\\n") as line> + ${line} + + +<#else> ${item.value} + \ No newline at end of file From 5a6d206158be36461fb00bea09e2f4b3ffafafe5 Mon Sep 17 00:00:00 2001 From: chenghongchen Date: Wed, 23 Oct 2024 10:58:15 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=20=E6=B7=BB=E5=8A=A0=20hadoop=20=E7=94=A8?= =?UTF-8?q?=E6=88=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/datasophon/worker/WorkerApplicationServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/WorkerApplicationServer.java b/datasophon-worker/src/main/java/com/datasophon/worker/WorkerApplicationServer.java index f8c06bff..9ff4bcb3 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/WorkerApplicationServer.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/WorkerApplicationServer.java @@ -106,6 +106,7 @@ public static void main(String[] args) throws UnknownHostException { } private static void initUserMap(Map userMap) { + userMap.put("hadoop", HADOOP); userMap.put("hdfs", HADOOP); userMap.put("yarn", HADOOP); userMap.put("hive", HADOOP);