diff --git a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java index 3e562bf9..99fdb709 100644 --- a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java +++ b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java @@ -50,9 +50,7 @@ public void onMessage(ReqT message) { if (isTaskRequest(message)) { try { Method method = message.getClass().getDeclaredMethod("getTaskId"); - method.setAccessible(true); Long taskId = (Long) method.invoke(message); - method.setAccessible(false); truncateLogFile(taskId); MDC.put("taskId", String.valueOf(taskId)); Caches.RUNNING_TASK = taskId; diff --git a/bigtop-manager-grpc/src/main/resources/proto/component_command.proto b/bigtop-manager-grpc/src/main/resources/proto/component_command.proto index 3b3d1c5c..180b8b8f 100644 --- a/bigtop-manager-grpc/src/main/resources/proto/component_command.proto +++ b/bigtop-manager-grpc/src/main/resources/proto/component_command.proto @@ -18,8 +18,6 @@ */ syntax = "proto3"; -import "google/protobuf/struct.proto"; - option java_multiple_files = true; option java_package = "org.apache.bigtop.manager.grpc.generated"; option java_outer_classname = "ComponentCommandProto"; diff --git a/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto b/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto index baa83104..adc44046 100644 --- a/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto +++ b/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto @@ -18,8 +18,6 @@ */ syntax = "proto3"; -import "google/protobuf/struct.proto"; - option java_multiple_files = true; option java_package = "org.apache.bigtop.manager.grpc.generated"; option java_outer_classname = "JobCacheProto"; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java index fb4ba777..7ffadc05 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java @@ -38,6 +38,7 @@ import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; import org.apache.bigtop.manager.server.utils.StackUtils; import java.util.ArrayList; @@ -169,8 +170,10 @@ private void saveService(ServiceCommandDTO serviceCommand) { // Persist current configs Map confMap = new HashMap<>(); - List configs = serviceCommand.getConfigs(); - List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(configs); + List oriConfigs = StackUtils.SERVICE_CONFIG_MAP.get(serviceName); + List newConfigs = serviceCommand.getConfigs(); + List mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs, newConfigs); + List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(mergedConfigs); for (ServiceConfigPO serviceConfigPO : serviceConfigPOList) { serviceConfigPO.setClusterId(clusterId); serviceConfigPO.setServiceId(servicePO.getId()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java index a3481239..703bf868 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java @@ -21,6 +21,7 @@ import org.apache.bigtop.manager.dao.po.ServiceConfigPO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; +import org.apache.bigtop.manager.server.model.req.ServiceConfigReq; import org.apache.bigtop.manager.server.model.vo.ServiceConfigVO; import org.mapstruct.Mapper; @@ -36,6 +37,10 @@ public interface ServiceConfigConverter { ServiceConfigConverter INSTANCE = Mappers.getMapper(ServiceConfigConverter.class); + ServiceConfigDTO fromReq2DTO(ServiceConfigReq serviceConfigReq); + + List fromReq2DTO(List serviceConfigReqList); + @Mapping(target = "properties", source = "propertiesJson", qualifiedByName = "json2PropertyDTOList") ServiceConfigDTO fromPO2DTO(ServiceConfigPO serviceConfigPO); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java index d9c86556..9c99ddbf 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java @@ -34,6 +34,7 @@ import org.apache.bigtop.manager.server.model.converter.ServiceConfigConverter; import org.apache.bigtop.manager.server.model.converter.ServiceConfigSnapshotConverter; import org.apache.bigtop.manager.server.model.converter.ServiceConverter; +import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; import org.apache.bigtop.manager.server.model.query.PageQuery; import org.apache.bigtop.manager.server.model.req.ServiceConfigReq; import org.apache.bigtop.manager.server.model.req.ServiceConfigSnapshotReq; @@ -43,6 +44,8 @@ import org.apache.bigtop.manager.server.model.vo.ServiceVO; import org.apache.bigtop.manager.server.service.ServiceService; import org.apache.bigtop.manager.server.utils.PageUtils; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; +import org.apache.bigtop.manager.server.utils.StackUtils; import org.apache.commons.collections4.CollectionUtils; @@ -54,7 +57,6 @@ import lombok.extern.slf4j.Slf4j; import jakarta.annotation.Resource; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -116,15 +118,26 @@ public List listConf(Long clusterId, Long serviceId) { @Override public List updateConf(Long clusterId, Long serviceId, List reqs) { - List list = new ArrayList<>(); - for (ServiceConfigReq req : reqs) { - ServiceConfigPO serviceConfigPO = new ServiceConfigPO(); - serviceConfigPO.setId(req.getId()); - serviceConfigPO.setPropertiesJson(JsonUtils.writeAsString(req.getProperties())); - list.add(serviceConfigPO); - } - - serviceConfigDao.partialUpdateByIds(list); + ServicePO servicePO = serviceDao.findById(serviceId); + List configs = serviceConfigDao.findByServiceId(serviceId); + + List oriConfigs; + List newConfigs; + List mergedConfigs; + + // Merge stack config with existing config first, in case new property has been added to stack config. + oriConfigs = StackUtils.SERVICE_CONFIG_MAP.get(servicePO.getName()); + newConfigs = ServiceConfigConverter.INSTANCE.fromPO2DTO(configs); + mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs, newConfigs); + + // Merge existing config with new config in request object + oriConfigs = mergedConfigs; + newConfigs = ServiceConfigConverter.INSTANCE.fromReq2DTO(reqs); + mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs, newConfigs); + + // Save merged config + List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(mergedConfigs); + serviceConfigDao.partialUpdateByIds(serviceConfigPOList); return listConf(clusterId, serviceId); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java index 64e800e4..430172e4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java @@ -20,12 +20,17 @@ import org.apache.bigtop.manager.server.model.dto.AttrsDTO; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; import org.apache.bigtop.manager.server.stack.model.AttrsModel; import org.apache.bigtop.manager.server.stack.model.PropertyModel; import org.apache.bigtop.manager.server.stack.xml.ConfigurationXml; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -68,23 +73,81 @@ private static PropertyDTO getPropertyDTO(PropertyModel propertyModel) { return propertyDTO; } + public static List mergeServiceConfigs( + List oriConfigs, List overrideConfigs) { + // To avoid to change the original configs, we use cloned object + List mergedConfigs = new ArrayList<>(); + for (ServiceConfigDTO oriConfig : oriConfigs) { + ServiceConfigDTO mergedConfig = new ServiceConfigDTO(); + BeanUtils.copyProperties(oriConfig, mergedConfig); + mergedConfigs.add(mergedConfig); + } + + if (CollectionUtils.isEmpty(overrideConfigs)) { + return mergedConfigs; + } + + // Assign id for each service config + for (ServiceConfigDTO config : mergedConfigs) { + config.setId(overrideConfigs.stream() + .filter(x -> x.getName().equals(config.getName())) + .findFirst() + .map(ServiceConfigDTO::getId) + .orElse(null)); + } + + Map> overrideConfigsMap = serviceConfig2Map(overrideConfigs); + for (ServiceConfigDTO mergedConfig : mergedConfigs) { + String configName = mergedConfig.getName(); + if (!overrideConfigsMap.containsKey(configName)) { + continue; + } + + // Override existing properties + Map overridePropertiesMap = overrideConfigsMap.get(configName); + for (PropertyDTO property : mergedConfig.getProperties()) { + String propertyName = property.getName(); + String value = overridePropertiesMap.remove(propertyName); + if (value != null) { + property.setValue(value); + } + } + + // We may still have some properties added by user manually + if (MapUtils.isNotEmpty(overridePropertiesMap)) { + for (Map.Entry entry : overridePropertiesMap.entrySet()) { + PropertyDTO property = new PropertyDTO(); + property.setName(entry.getKey()); + property.setValue(entry.getValue()); + mergedConfig.getProperties().add(property); + } + } + } + + return mergedConfigs; + } + /** - * extract config from List> to Map + * extract config from List to Map> * - * @param list List - * @return Map + * @param configs List + * @return Map> */ - public static Map extractConfigMap(List list) { - if (list == null) { - return null; + private static Map> serviceConfig2Map(List configs) { + Map> outerMap = new HashMap<>(); + if (CollectionUtils.isEmpty(configs)) { + return outerMap; } - Map hashMap = new HashMap<>(); - for (PropertyDTO property : list) { - String key = property.getName(); - Object value = property.getValue(); - hashMap.put(key, value); + for (ServiceConfigDTO config : configs) { + Map innerMap = new HashMap<>(); + for (PropertyDTO property : config.getProperties()) { + innerMap.put(property.getName(), property.getValue()); + } + + outerMap.put(config.getName(), innerMap); } - return hashMap; + + return outerMap; } }