BIGTOP-4294: Install ZooKeeper with tarball (#117)

apache · Dec 6, 2024 · 8a37894 · 8a37894
1 parent 42a1721
commit 8a37894
Show file tree

Hide file tree

Showing 20 changed files with 102 additions and 102 deletions.
diff --git a/bigtop-manager-bom/pom.xml b/bigtop-manager-bom/pom.xml
@@ -34,7 +34,7 @@
         <spring-boot.version>3.1.1</spring-boot.version>
         <springdoc.version>2.2.0</springdoc.version>
         <freemarker.version>2.3.32</freemarker.version>
-        <common-lang3.version>3.12.0</common-lang3.version>
+        <common-lang3.version>3.14.0</common-lang3.version>
         <snakeyaml.version>2.0</snakeyaml.version>
         <commons-io.version>2.16.1</commons-io.version>
         <guava.version>32.1.1-jre</guava.version>

diff --git a/...op-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java b/...op-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java
@@ -24,33 +24,13 @@ private Constants() {
         throw new UnsupportedOperationException("Construct Constants");
     }
 
-    /**
-     * host key for all hosts
-     */
     public static final String ALL_HOST_KEY = "all";
 
-    /**
-     * permission 644
-     */
     public static final String PERMISSION_644 = "644";
 
-    /**
-     * permission 755
-     */
     public static final String PERMISSION_755 = "755";
 
-    /**
-     * permission 775
-     */
-    public static final String PERMISSION_775 = "775";
-
-    /**
-     * permission 777
-     */
     public static final String PERMISSION_777 = "777";
 
-    /**
-     * root user
-     */
     public static final String ROOT_USER = "root";
 }
diff --git a/...rver/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java b/...rver/src/main/java/org/apache/bigtop/manager/server/command/task/CacheFileUpdateTask.java
@@ -118,13 +118,17 @@ private void genFullCaches() {
 
         serviceConfigMap = new HashMap<>();
         for (ServiceConfigPO serviceConfigPO : serviceConfigPOList) {
+            List<Map<String, Object>> properties = JsonUtils.readFromString(serviceConfigPO.getPropertiesJson());
+            Map<String, String> kvMap = properties.stream()
+                    .collect(Collectors.toMap(
+                            property -> (String) property.get("name"), property -> (String) property.get("value")));
+            String kvString = JsonUtils.writeAsString(kvMap);
+
             if (serviceConfigMap.containsKey(serviceConfigPO.getServiceName())) {
-                serviceConfigMap
-                        .get(serviceConfigPO.getServiceName())
-                        .put(serviceConfigPO.getName(), serviceConfigPO.getPropertiesJson());
+                serviceConfigMap.get(serviceConfigPO.getServiceName()).put(serviceConfigPO.getName(), kvString);
             } else {
                 Map<String, Object> hashMap = new HashMap<>();
-                hashMap.put(serviceConfigPO.getName(), serviceConfigPO.getPropertiesJson());
+                hashMap.put(serviceConfigPO.getName(), kvString);
                 serviceConfigMap.put(serviceConfigPO.getServiceName(), hashMap);
             }
         }

diff --git a/...r-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java b/...r-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java
@@ -218,7 +218,7 @@ public Boolean installDependencies(List<Long> hostIds, String path) {
             }
 
             // Run agent in background
-            command = "nohup " + path + "/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &";
+            command = "nohup " + path + "/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &";
             result = execCommandOnRemoteHost(hostDTO, hostDTO.getHostname(), command);
             if (result.getExitCode() != MessageConstants.SUCCESS_CODE) {
                 hostPO.setErrInfo(result.getErrMsg());

diff --git a/...ger-server/src/main/java/org/apache/bigtop/manager/server/timer/ComponentStatusTimer.java b/...ger-server/src/main/java/org/apache/bigtop/manager/server/timer/ComponentStatusTimer.java
@@ -54,6 +54,10 @@ public class ComponentStatusTimer {
     public void execute() {
         List<ComponentPO> componentPOList = componentDao.findAll();
         for (ComponentPO componentPO : componentPOList) {
+            if (HealthyStatusEnum.fromCode(componentPO.getStatus()) == HealthyStatusEnum.UNKNOWN) {
+                continue;
+            }
+
             componentPO = componentDao.findDetailsById(componentPO.getId());
             HostPO hostPO = hostDao.findById(componentPO.getHostId());
             ComponentStatusRequest request = ComponentStatusRequest.newBuilder()

diff --git a/...erver/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml b/...erver/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml
@@ -45,12 +45,6 @@
         <display-name>Port for running ZK Server</display-name>
         <description>Port for running ZK Server.</description>
     </property>
-    <property>
-        <name>dataDir</name>
-        <value>/hadoop/zookeeper</value>
-        <display-name>ZooKeeper directory</display-name>
-        <description>Data directory for ZooKeeper.</description>
-    </property>
     <property>
         <name>autopurge.snapRetainCount</name>
         <value>30</value>

diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
@@ -54,7 +54,7 @@
                 <packages>
                     <package>
                         <name>zookeeper-3.7.2-1.tar.gz</name>
-                        <checksum>380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3</checksum>
+                        <checksum>SHA-256:380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3</checksum>
                     </package>
                 </packages>
             </package-specific>

diff --git a/...ck-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java b/...ck-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
@@ -83,7 +83,7 @@ public static ShellResult config(Params params, String componentName) {
         LinuxFileUtils.createDirectories(
                 hdfsParams.getDfsDataDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true);
         LinuxFileUtils.createDirectories(
-                hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_775, true);
+                hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true);
         LinuxFileUtils.createDirectories(
                 hdfsParams.getHadoopPidDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true);
 

diff --git a/...n/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java b/...n/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java
@@ -26,10 +26,20 @@
 import com.google.auto.service.AutoService;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.Properties;
+
 @Slf4j
 @AutoService(Script.class)
 public class ZookeeperClientScript extends AbstractClientScript {
 
+    @Override
+    public ShellResult add(Params params) {
+        Properties properties = new Properties();
+        properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1");
+
+        return super.add(params, properties);
+    }
+
     @Override
     public ShellResult configure(Params params) {
         return ZookeeperSetup.config(params);

diff --git a/...rc/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java b/...rc/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java
@@ -31,10 +31,10 @@
 @Getter
 public class ZookeeperParams extends BigtopParams {
 
-    private String zookeeperLogDir = "/var/log/zookeeper";
-    private String zookeeperPidDir = "/var/run/zookeeper";
-    private String zookeeperDataDir = "/hadoop/zookeeper";
-    private String zookeeperPidFile = zookeeperPidDir + "/zookeeper_server.pid";
+    private final String zookeeperLogDir = "/var/log/zookeeper";
+    private final String zookeeperPidDir = "/var/run/zookeeper";
+    private final String zookeeperDataDir = serviceHome() + "/data";
+    private final String zookeeperPidFile = zookeeperPidDir + "/zookeeper_server.pid";
 
     public ZookeeperParams(CommandPayload commandPayload) {
         super(commandPayload);
@@ -46,17 +46,13 @@ public ZookeeperParams(CommandPayload commandPayload) {
 
     @GlobalParams
     public Map<String, Object> zooCfg() {
-        Map<String, Object> zooCfg = LocalSettings.configurations(serviceName(), "zoo.cfg");
-        zookeeperDataDir = (String) zooCfg.get("dataDir");
-        return zooCfg;
+        Map<String, Object> configurations = LocalSettings.configurations(serviceName(), "zoo.cfg");
+        configurations.put("dataDir", zookeeperDataDir);
+        return configurations;
     }
 
     @GlobalParams
     public Map<String, Object> zookeeperEnv() {
-        Map<String, Object> zookeeperEnv = LocalSettings.configurations(serviceName(), "zookeeper-env");
-        zookeeperLogDir = (String) zookeeperEnv.get("zookeeper_log_dir");
-        zookeeperPidDir = (String) zookeeperEnv.get("zookeeper_pid_dir");
-        zookeeperPidFile = zookeeperPidDir + "/zookeeper_server.pid";
-        return zookeeperEnv;
+        return LocalSettings.configurations(serviceName(), "zookeeper-env");
     }
 }
diff --git a/...n/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java b/...n/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java
@@ -30,11 +30,20 @@
 
 import java.io.IOException;
 import java.text.MessageFormat;
+import java.util.Properties;
 
 @Slf4j
 @AutoService(Script.class)
 public class ZookeeperServerScript extends AbstractServerScript {
 
+    @Override
+    public ShellResult add(Params params) {
+        Properties properties = new Properties();
+        properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1");
+
+        return super.add(params, properties);
+    }
+
     @Override
     public ShellResult configure(Params params) {
         return ZookeeperSetup.config(params);

diff --git a/...src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperSetup.java b/...src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperSetup.java
@@ -18,6 +18,7 @@
  */
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.zookeeper;
 
+import org.apache.bigtop.manager.common.constants.Constants;
 import org.apache.bigtop.manager.common.shell.ShellResult;
 import org.apache.bigtop.manager.common.utils.NetUtils;
 import org.apache.bigtop.manager.stack.core.enums.ConfigType;
@@ -34,9 +35,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644;
-import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;
-
 @Slf4j
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class ZookeeperSetup {
@@ -53,13 +51,12 @@ public static ShellResult config(Params params) {
         List<String> zkHostList = LocalSettings.hosts("zookeeper_server");
 
         LinuxFileUtils.createDirectories(
-                zookeeperParams.getZookeeperDataDir(), zookeeperUser, zookeeperGroup, PERMISSION_755, true);
+                zookeeperParams.getZookeeperDataDir(), zookeeperUser, zookeeperGroup, Constants.PERMISSION_755, true);
         LinuxFileUtils.createDirectories(
-                zookeeperParams.getZookeeperLogDir(), zookeeperUser, zookeeperGroup, PERMISSION_755, true);
+                zookeeperParams.getZookeeperLogDir(), zookeeperUser, zookeeperGroup, Constants.PERMISSION_755, true);
         LinuxFileUtils.createDirectories(
-                zookeeperParams.getZookeeperPidDir(), zookeeperUser, zookeeperGroup, PERMISSION_755, true);
+                zookeeperParams.getZookeeperPidDir(), zookeeperUser, zookeeperGroup, Constants.PERMISSION_755, true);
 
-        // 针对zkHostList排序，获取当前hostname的index+1
         // server.${host?index+1}=${host}:2888:3888
         zkHostList.sort(String::compareToIgnoreCase);
         StringBuilder zkServerStr = new StringBuilder();
@@ -75,7 +72,7 @@ public static ShellResult config(Params params) {
                 MessageFormat.format("{0}/myid", zookeeperParams.getZookeeperDataDir()),
                 zookeeperUser,
                 zookeeperGroup,
-                PERMISSION_644,
+                Constants.PERMISSION_644,
                 zkHostList.indexOf(NetUtils.getHostname()) + 1 + "");
 
         // zoo.cfg
@@ -87,7 +84,7 @@ public static ShellResult config(Params params) {
                 MessageFormat.format("{0}/zoo.cfg", confDir),
                 zookeeperUser,
                 zookeeperGroup,
-                PERMISSION_644,
+                Constants.PERMISSION_644,
                 Map.of("model", map),
                 paramMap);
 
@@ -97,7 +94,7 @@ public static ShellResult config(Params params) {
                 MessageFormat.format("{0}/zookeeper-env.sh", confDir),
                 zookeeperUser,
                 zookeeperGroup,
-                PERMISSION_644,
+                Constants.PERMISSION_644,
                 zookeeperParams.getGlobalParamsMap());
 
         return ShellResult.success("ZooKeeper Server Configure success!");

diff --git a/...nager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java b/...nager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java
@@ -81,7 +81,7 @@ public String hostname() {
      */
     @Override
     public String confDir() {
-        return "/etc/" + this.commandPayload.getServiceName().toLowerCase() + "/conf";
+        return serviceHome() + "/conf";
     }
 
     @Override

diff --git a/...ck-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/...ck-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java
@@ -24,7 +24,6 @@
 import org.apache.bigtop.manager.common.shell.ShellResult;
 import org.apache.bigtop.manager.stack.core.param.Params;
 import org.apache.bigtop.manager.stack.core.utils.TarballUtils;
-import org.apache.bigtop.manager.stack.core.utils.linux.LinuxAccountUtils;
 import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils;
 
 import org.apache.commons.lang3.StringUtils;
@@ -54,17 +53,14 @@ public ShellResult add(Params params, Properties properties) {
 
         if (!Files.exists(Path.of(stackHome))) {
             String user = System.getProperty("user.name");
-            String group = params.group();
-
-            LinuxAccountUtils.assignUserToSupGroups(user, List.of(group));
-            LinuxFileUtils.createDirectories(stackHome, user, group, Constants.PERMISSION_755, true);
+            LinuxFileUtils.createDirectories(stackHome, user, user, Constants.PERMISSION_755, true);
         }
 
         for (PackageInfo packageInfo : packages) {
             Integer skipLevels = Integer.parseInt(properties.getProperty(PROPERTY_KEY_SKIP_LEVELS, "0"));
             TarballUtils.installPackage(repo.getBaseUrl(), stackHome, serviceHome, packageInfo, skipLevels);
 
-            // Dir already created by TarballUtils, this changes the owner and permission for the service
+            // Dir already created by TarballUtils, this changes the owner and permission for the service home
             LinuxFileUtils.createDirectories(
                     serviceHome, params.user(), params.group(), Constants.PERMISSION_755, true);
         }

diff --git a/...er-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/LocalSettings.java b/...er-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/LocalSettings.java
@@ -51,7 +51,7 @@ public static Map<String, Object> configurations(String service, String type) {
                 Object configData =
                         configJson.getOrDefault(service, new HashMap<>()).get(type);
                 if (configData != null) {
-                    configDataMap = JsonUtils.readFromString((String) configData, new TypeReference<>() {});
+                    configDataMap = JsonUtils.readFromString(configData.toString(), new TypeReference<>() {});
                 }
             }
         } catch (Exception e) {

diff --git a/...ger-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java b/...ger-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java
@@ -23,7 +23,6 @@
 import org.apache.bigtop.manager.stack.core.tarball.ChecksumValidator;
 import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;
 import org.apache.bigtop.manager.stack.core.tarball.TarballExtractor;
-import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -38,8 +37,8 @@ public class TarballUtils {
     public static void installPackage(
             String repoUrl, String stackHome, String serviceHome, PackageInfo packageInfo, Integer skipLevels) {
         if (Files.exists(Path.of(serviceHome))) {
-            log.info("Service home [{}] exists, deleting...", serviceHome);
-            LinuxFileUtils.removeDirectories(serviceHome);
+            log.info("Service home [{}] exists, skip downloading...", serviceHome);
+            return;
         }
 
         String remoteUrl = repoUrl + File.separator + packageInfo.getName();

diff --git a/...ore/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxAccountUtils.java b/...ore/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxAccountUtils.java
@@ -246,33 +246,6 @@ public static void groupAdd(String group, String gid, String password) {
         }
     }
 
-    /**
-     * Assign user to supplementary groups
-     *
-     * @param user user
-     * @param groups groups
-     */
-    public static void assignUserToSupGroups(String user, List<String> groups) {
-        Objects.requireNonNull(user);
-        Objects.requireNonNull(groups);
-
-        List<String> builderParameters = new ArrayList<>();
-
-        builderParameters.add("usermod");
-        builderParameters.add("-aG");
-        builderParameters.add(String.join(",", groups));
-        builderParameters.add(user);
-
-        try {
-            ShellResult shellResult = sudoExecCmd(builderParameters);
-            if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) {
-                throw new StackException(shellResult.getErrMsg());
-            }
-        } catch (IOException e) {
-            throw new StackException(e);
-        }
-    }
-
     /**
      * Check if exists group
      *