From e3d34933d526410a149621fbd2932b95b77378e5 Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Sat, 25 Jan 2025 15:27:00 +0800 Subject: [PATCH 1/2] merge config --- .../grpc/interceptor/TaskInterceptor.java | 2 - .../resources/proto/component_command.proto | 2 - .../src/main/resources/proto/job_cache.proto | 2 - .../command/job/service/ServiceAddJob.java | 7 +- .../converter/ServiceConfigConverter.java | 5 ++ .../service/impl/ServiceServiceImpl.java | 32 +++++--- .../server/utils/StackConfigUtils.java | 78 ++++++++++++++++--- 7 files changed, 99 insertions(+), 29 deletions(-) diff --git a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java index 3e562bf9d..99fdb709a 100644 --- a/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java +++ b/bigtop-manager-agent/src/main/java/org/apache/bigtop/manager/agent/grpc/interceptor/TaskInterceptor.java @@ -50,9 +50,7 @@ public void onMessage(ReqT message) { if (isTaskRequest(message)) { try { Method method = message.getClass().getDeclaredMethod("getTaskId"); - method.setAccessible(true); Long taskId = (Long) method.invoke(message); - method.setAccessible(false); truncateLogFile(taskId); MDC.put("taskId", String.valueOf(taskId)); Caches.RUNNING_TASK = taskId; diff --git a/bigtop-manager-grpc/src/main/resources/proto/component_command.proto b/bigtop-manager-grpc/src/main/resources/proto/component_command.proto index 3b3d1c5c3..180b8b8f9 100644 --- a/bigtop-manager-grpc/src/main/resources/proto/component_command.proto +++ b/bigtop-manager-grpc/src/main/resources/proto/component_command.proto @@ -18,8 +18,6 @@ */ syntax = "proto3"; -import "google/protobuf/struct.proto"; - option java_multiple_files = true; option java_package = "org.apache.bigtop.manager.grpc.generated"; option java_outer_classname = "ComponentCommandProto"; diff --git a/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto b/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto index baa831045..adc44046b 100644 --- a/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto +++ b/bigtop-manager-grpc/src/main/resources/proto/job_cache.proto @@ -18,8 +18,6 @@ */ syntax = "proto3"; -import "google/protobuf/struct.proto"; - option java_multiple_files = true; option java_package = "org.apache.bigtop.manager.grpc.generated"; option java_outer_classname = "JobCacheProto"; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java index fb4ba7771..7ffadc055 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/service/ServiceAddJob.java @@ -38,6 +38,7 @@ import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; import org.apache.bigtop.manager.server.utils.StackUtils; import java.util.ArrayList; @@ -169,8 +170,10 @@ private void saveService(ServiceCommandDTO serviceCommand) { // Persist current configs Map confMap = new HashMap<>(); - List configs = serviceCommand.getConfigs(); - List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(configs); + List oriConfigs = StackUtils.SERVICE_CONFIG_MAP.get(serviceName); + List newConfigs = serviceCommand.getConfigs(); + List mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs, newConfigs); + List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(mergedConfigs); for (ServiceConfigPO serviceConfigPO : serviceConfigPOList) { serviceConfigPO.setClusterId(clusterId); serviceConfigPO.setServiceId(servicePO.getId()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java index a34812397..703bf8680 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConfigConverter.java @@ -21,6 +21,7 @@ import org.apache.bigtop.manager.dao.po.ServiceConfigPO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; +import org.apache.bigtop.manager.server.model.req.ServiceConfigReq; import org.apache.bigtop.manager.server.model.vo.ServiceConfigVO; import org.mapstruct.Mapper; @@ -36,6 +37,10 @@ public interface ServiceConfigConverter { ServiceConfigConverter INSTANCE = Mappers.getMapper(ServiceConfigConverter.class); + ServiceConfigDTO fromReq2DTO(ServiceConfigReq serviceConfigReq); + + List fromReq2DTO(List serviceConfigReqList); + @Mapping(target = "properties", source = "propertiesJson", qualifiedByName = "json2PropertyDTOList") ServiceConfigDTO fromPO2DTO(ServiceConfigPO serviceConfigPO); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java index d9c865569..70b6f115f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java @@ -34,6 +34,7 @@ import org.apache.bigtop.manager.server.model.converter.ServiceConfigConverter; import org.apache.bigtop.manager.server.model.converter.ServiceConfigSnapshotConverter; import org.apache.bigtop.manager.server.model.converter.ServiceConverter; +import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; import org.apache.bigtop.manager.server.model.query.PageQuery; import org.apache.bigtop.manager.server.model.req.ServiceConfigReq; import org.apache.bigtop.manager.server.model.req.ServiceConfigSnapshotReq; @@ -44,6 +45,8 @@ import org.apache.bigtop.manager.server.service.ServiceService; import org.apache.bigtop.manager.server.utils.PageUtils; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; +import org.apache.bigtop.manager.server.utils.StackUtils; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; @@ -116,15 +119,26 @@ public List listConf(Long clusterId, Long serviceId) { @Override public List updateConf(Long clusterId, Long serviceId, List reqs) { - List list = new ArrayList<>(); - for (ServiceConfigReq req : reqs) { - ServiceConfigPO serviceConfigPO = new ServiceConfigPO(); - serviceConfigPO.setId(req.getId()); - serviceConfigPO.setPropertiesJson(JsonUtils.writeAsString(req.getProperties())); - list.add(serviceConfigPO); - } - - serviceConfigDao.partialUpdateByIds(list); + ServicePO servicePO = serviceDao.findById(serviceId); + List configs = serviceConfigDao.findByServiceId(serviceId); + + List oriConfigs; + List newConfigs; + List mergedConfigs; + + // Merge stack config with existing config first, in case new property has been added to stack config. + oriConfigs = StackUtils.SERVICE_CONFIG_MAP.get(servicePO.getName()); + newConfigs = ServiceConfigConverter.INSTANCE.fromPO2DTO(configs); + mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs, newConfigs); + + // Merge existing config with new config in request object + oriConfigs = mergedConfigs; + newConfigs = ServiceConfigConverter.INSTANCE.fromReq2DTO(reqs); + mergedConfigs = StackConfigUtils.mergeServiceConfigs(oriConfigs, newConfigs); + + // Save merged config + List serviceConfigPOList = ServiceConfigConverter.INSTANCE.fromDTO2PO(mergedConfigs); + serviceConfigDao.partialUpdateByIds(serviceConfigPOList); return listConf(clusterId, serviceId); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java index 64e800e44..16e061c69 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java @@ -18,13 +18,19 @@ */ package org.apache.bigtop.manager.server.utils; +import dev.langchain4j.agent.tool.P; +import org.apache.bigtop.manager.dao.repository.ServiceConfigDao; import org.apache.bigtop.manager.server.model.dto.AttrsDTO; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; import org.apache.bigtop.manager.server.stack.model.AttrsModel; import org.apache.bigtop.manager.server.stack.model.PropertyModel; import org.apache.bigtop.manager.server.stack.xml.ConfigurationXml; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; import java.util.ArrayList; import java.util.HashMap; @@ -68,23 +74,71 @@ private static PropertyDTO getPropertyDTO(PropertyModel propertyModel) { return propertyDTO; } + public static List mergeServiceConfigs(List oriConfigs, List overrideConfigs) { + // To avoid to change the original configs, we use cloned object + List mergedConfigs = new ArrayList<>(); + for (ServiceConfigDTO oriConfig : oriConfigs) { + ServiceConfigDTO mergedConfig = new ServiceConfigDTO(); + BeanUtils.copyProperties(oriConfig, mergedConfig); + mergedConfigs.add(mergedConfig); + } + + if (CollectionUtils.isEmpty(overrideConfigs)) { + return mergedConfigs; + } + + Map> overrideConfigsMap = serviceConfig2Map(overrideConfigs); + for (ServiceConfigDTO mergedConfig : mergedConfigs) { + String configName = mergedConfig.getName(); + if (!overrideConfigsMap.containsKey(configName)) { + continue; + } + + // Override existing properties + Map overridePropertiesMap = overrideConfigsMap.get(configName); + for (PropertyDTO property : mergedConfig.getProperties()) { + String propertyName = property.getName(); + String value = overridePropertiesMap.remove(propertyName); + if (value != null) { + property.setValue(value); + } + } + + // We still have some properties, maybe added by user manually + if (MapUtils.isNotEmpty(overridePropertiesMap)) { + for (Map.Entry entry : overridePropertiesMap.entrySet()) { + PropertyDTO property = new PropertyDTO(); + property.setName(entry.getKey()); + property.setValue(entry.getValue()); + mergedConfig.getProperties().add(property); + } + } + } + + return mergedConfigs; + } + /** - * extract config from List> to Map + * extract config from List to Map> * - * @param list List - * @return Map + * @param configs List + * @return Map> */ - public static Map extractConfigMap(List list) { - if (list == null) { - return null; + private static Map> serviceConfig2Map(List configs) { + Map> outerMap = new HashMap<>(); + if (CollectionUtils.isEmpty(configs)) { + return outerMap; } - Map hashMap = new HashMap<>(); - for (PropertyDTO property : list) { - String key = property.getName(); - Object value = property.getValue(); - hashMap.put(key, value); + for (ServiceConfigDTO config : configs) { + Map innerMap = new HashMap<>(); + for (PropertyDTO property : config.getProperties()) { + innerMap.put(property.getName(), property.getValue()); + } + + outerMap.put(config.getName(), innerMap); } - return hashMap; + + return outerMap; } } From b8022b5c3446db4cda7a687209989feb8d499a46 Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Sat, 25 Jan 2025 23:49:51 +0800 Subject: [PATCH 2/2] codestyle --- .../server/service/impl/ServiceServiceImpl.java | 3 +-- .../manager/server/utils/StackConfigUtils.java | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java index 70b6f115f..9c99ddbff 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java @@ -44,9 +44,9 @@ import org.apache.bigtop.manager.server.model.vo.ServiceVO; import org.apache.bigtop.manager.server.service.ServiceService; import org.apache.bigtop.manager.server.utils.PageUtils; - import org.apache.bigtop.manager.server.utils.StackConfigUtils; import org.apache.bigtop.manager.server.utils.StackUtils; + import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; @@ -57,7 +57,6 @@ import lombok.extern.slf4j.Slf4j; import jakarta.annotation.Resource; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java index 16e061c69..430172e4d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackConfigUtils.java @@ -18,8 +18,6 @@ */ package org.apache.bigtop.manager.server.utils; -import dev.langchain4j.agent.tool.P; -import org.apache.bigtop.manager.dao.repository.ServiceConfigDao; import org.apache.bigtop.manager.server.model.dto.AttrsDTO; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; @@ -30,6 +28,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; + import org.springframework.beans.BeanUtils; import java.util.ArrayList; @@ -74,7 +73,8 @@ private static PropertyDTO getPropertyDTO(PropertyModel propertyModel) { return propertyDTO; } - public static List mergeServiceConfigs(List oriConfigs, List overrideConfigs) { + public static List mergeServiceConfigs( + List oriConfigs, List overrideConfigs) { // To avoid to change the original configs, we use cloned object List mergedConfigs = new ArrayList<>(); for (ServiceConfigDTO oriConfig : oriConfigs) { @@ -87,6 +87,15 @@ public static List mergeServiceConfigs(List return mergedConfigs; } + // Assign id for each service config + for (ServiceConfigDTO config : mergedConfigs) { + config.setId(overrideConfigs.stream() + .filter(x -> x.getName().equals(config.getName())) + .findFirst() + .map(ServiceConfigDTO::getId) + .orElse(null)); + } + Map> overrideConfigsMap = serviceConfig2Map(overrideConfigs); for (ServiceConfigDTO mergedConfig : mergedConfigs) { String configName = mergedConfig.getName(); @@ -104,7 +113,7 @@ public static List mergeServiceConfigs(List } } - // We still have some properties, maybe added by user manually + // We may still have some properties added by user manually if (MapUtils.isNotEmpty(overridePropertiesMap)) { for (Map.Entry entry : overridePropertiesMap.entrySet()) { PropertyDTO property = new PropertyDTO();