diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index c16144ffe..675e7cb54 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -53,6 +53,8 @@ public static void addServlets(ServletContextHandler context) { context.addServlet(ListRecommendations.class, ServerContext.RECOMMEND_RESULTS); context.addServlet(PerformanceProfileService.class, ServerContext.CREATE_PERF_PROFILE); context.addServlet(PerformanceProfileService.class, ServerContext.LIST_PERF_PROFILES); + context.addServlet(ListDatasources.class, ServerContext.LIST_DATASOURCES); + context.addServlet(DSMetadataService.class, ServerContext.DATASOURCE_METADATA); // Adding UI support API's context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java b/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java new file mode 100644 index 000000000..c48675d6f --- /dev/null +++ b/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2023 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +package com.autotune.analyzer.serviceObjects; + +import com.autotune.utils.KruizeConstants; +import com.google.gson.annotations.SerializedName; + +public class DSMetadataAPIObject { + private String version; + @SerializedName(KruizeConstants.JSONKeys.DATASOURCE_NAME) + private String dataSourceName; + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getDataSourceName() { + return dataSourceName; + } + public boolean validateInputFields() { + if (version == null || version.isEmpty() || dataSourceName == null || dataSourceName.isEmpty()) { + throw new IllegalArgumentException("Invalid input fields: version and datasource_name cannot be null or empty"); + } + return true; + } + +} diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/ListDatasourcesAPIObject.java b/src/main/java/com/autotune/analyzer/serviceObjects/ListDatasourcesAPIObject.java new file mode 100644 index 000000000..2c8a8ed48 --- /dev/null +++ b/src/main/java/com/autotune/analyzer/serviceObjects/ListDatasourcesAPIObject.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright (c) 2023 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +package com.autotune.analyzer.serviceObjects; + +import com.autotune.common.datasource.DataSourceInfo; +import com.autotune.utils.KruizeConstants; +import com.google.gson.annotations.SerializedName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class ListDatasourcesAPIObject { + private static final Logger LOGGER = LoggerFactory.getLogger(ListDatasourcesAPIObject.class); + @SerializedName(KruizeConstants.JSONKeys.VERSION) + private String apiversion = KruizeConstants.DataSourceConstants.DataSourceDetailsInfoConstants.version; + + @SerializedName(KruizeConstants.JSONKeys.DATASOURCES) + List dataSourceInfoList; + + public List getDataSourceInfoList() { + return dataSourceInfoList; + } + + public void setDataSourceInfoList(List dataSourceInfoList) { + if (null == dataSourceInfoList) { + LOGGER.error("No datasources found"); + return; + } + this.dataSourceInfoList = dataSourceInfoList; + } +} diff --git a/src/main/java/com/autotune/analyzer/services/DSMetadataService.java b/src/main/java/com/autotune/analyzer/services/DSMetadataService.java new file mode 100644 index 000000000..16594890e --- /dev/null +++ b/src/main/java/com/autotune/analyzer/services/DSMetadataService.java @@ -0,0 +1,286 @@ +/******************************************************************************* + * Copyright (c) 2023 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +package com.autotune.analyzer.services; + +import com.autotune.analyzer.serviceObjects.DSMetadataAPIObject; +import com.autotune.analyzer.utils.AnalyzerConstants; +import com.autotune.analyzer.utils.AnalyzerErrorConstants; +import com.autotune.analyzer.utils.GsonUTCDateAdapter; +import com.autotune.common.data.dataSourceMetadata.DataSourceMetadataInfo; +import com.autotune.common.datasource.DataSourceInfo; +import com.autotune.common.datasource.DataSourceManager; +import com.autotune.database.service.ExperimentDBService; +import com.autotune.utils.KruizeSupportedTypes; +import com.autotune.utils.MetricsConfig; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.micrometer.core.instrument.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.CHARACTER_ENCODING; +import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.JSON_CONTENT_TYPE; + +public class DSMetadataService extends HttpServlet { + private static final Logger LOGGER = LoggerFactory.getLogger(DSMetadataService.class); + + @Override + public void init(ServletConfig config) throws ServletException { + super.init(config); + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String statusValue = "failure"; + Timer.Sample timerImportDSMetadata = Timer.start(MetricsConfig.meterRegistry()); + //Key = dataSourceName + HashMap dataSourceMetadataMap = new HashMap<>(); + String inputData = ""; + + try { + // Set the character encoding of the request to UTF-8 + request.setCharacterEncoding(CHARACTER_ENCODING); + + inputData = request.getReader().lines().collect(Collectors.joining()); + + if (null == inputData || inputData.isEmpty()) { + throw new Exception("Request input data cannot be null or empty"); + } + + DSMetadataAPIObject metadataAPIObject = new Gson().fromJson(inputData, DSMetadataAPIObject.class); + + metadataAPIObject.validateInputFields(); + + String dataSourceName = metadataAPIObject.getDataSourceName(); + + if (null == dataSourceName || dataSourceName.isEmpty()) { + sendErrorResponse( + inputData, + response, + null, + HttpServletResponse.SC_BAD_REQUEST, + AnalyzerErrorConstants.APIErrors.DSMetadataAPI.DATASOURCE_NAME_MANDATORY); + } + + DataSourceInfo datasource = new ExperimentDBService().loadDataSourceFromDBByName(dataSourceName); + if(null != datasource) { + new DataSourceManager().importMetadataFromDataSource(datasource); + DataSourceMetadataInfo dataSourceMetadata = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName, "false"); + dataSourceMetadataMap.put(dataSourceName,dataSourceMetadata); + } + + if (dataSourceMetadataMap.isEmpty() || !dataSourceMetadataMap.containsKey(dataSourceName)) { + sendErrorResponse( + inputData, + response, + new Exception(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_DATASOURCE_NAME_METADATA_EXCPTN), + HttpServletResponse.SC_BAD_REQUEST, + String.format(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.DATASOURCE_METADATA_IMPORT_ERROR_MSG, dataSourceName) + ); + } else { + sendSuccessResponse(response, dataSourceMetadataMap.get(dataSourceName)); + } + } catch (Exception e) { + e.printStackTrace(); + LOGGER.error("Unknown exception caught: " + e.getMessage()); + sendErrorResponse(inputData, response, e, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Internal Server Error: " + e.getMessage()); + } finally { + if (null != timerImportDSMetadata) { + MetricsConfig.timerImportDSMetadata = MetricsConfig.timerBImportDSMetadata.tag("status", statusValue).register(MetricsConfig.meterRegistry()); + timerImportDSMetadata.stop(MetricsConfig.timerImportDSMetadata); + } + } + + } + + private void sendSuccessResponse(HttpServletResponse response, DataSourceMetadataInfo dataSourceMetadata) throws IOException { + response.setContentType(JSON_CONTENT_TYPE); + response.setCharacterEncoding(CHARACTER_ENCODING); + response.setStatus(HttpServletResponse.SC_CREATED); + + String gsonStr = ""; + if (null != dataSourceMetadata) { + Gson gsonObj = new GsonBuilder() + .disableHtmlEscaping() + .setPrettyPrinting() + .enableComplexMapKeySerialization() + .registerTypeAdapter(Date.class, new GsonUTCDateAdapter()) + .create(); + gsonStr = gsonObj.toJson(dataSourceMetadata); + } + response.getWriter().println(gsonStr); + response.getWriter().close(); + } + + public void sendErrorResponse(String inputRequestPayload, HttpServletResponse response, Exception e, int httpStatusCode, String errorMsg) throws + IOException { + if (null != e) { + LOGGER.error(e.toString()); + e.printStackTrace(); + if (null == errorMsg) errorMsg = e.getMessage(); + } + // check for the input request data to debug issues, if any + LOGGER.debug(inputRequestPayload); + response.sendError(httpStatusCode, errorMsg); + } + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException{ + String statusValue = "failure"; + Timer.Sample timerImportDSMetadata = Timer.start(MetricsConfig.meterRegistry()); + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType(JSON_CONTENT_TYPE); + response.setCharacterEncoding(CHARACTER_ENCODING); + String gsonStr; + + String dataSourceName = request.getParameter(AnalyzerConstants.ServiceConstants.DATASOURCE); + String clusterName = request.getParameter(AnalyzerConstants.ServiceConstants.CLUSTER_NAME); + String namespace = request.getParameter(AnalyzerConstants.ServiceConstants.NAMESPACE); + String verbose = request.getParameter(AnalyzerConstants.ServiceConstants.VERBOSE); + String internalVerbose = "false"; + //Key = dataSource name + HashMap dataSourceMetadataMap = new HashMap<>(); + boolean error = false; + // validate Query params + Set invalidParams = new HashSet<>(); + for (String param : request.getParameterMap().keySet()) { + if (!KruizeSupportedTypes.DSMETADATA_QUERY_PARAMS_SUPPORTED.contains(param)) { + invalidParams.add(param); + } + } + + try { + if (invalidParams.isEmpty()){ + if (null != verbose) { + internalVerbose = verbose; + } + + if (isValidBooleanValue(internalVerbose)) { + try { + if (null != dataSourceName) { + try { + DataSourceMetadataInfo dataSourceMetadata = null; + if (null == clusterName) { + dataSourceMetadata = new ExperimentDBService().loadMetadataFromDBByName(dataSourceName, internalVerbose); + + } else if (null != clusterName){ + if (null == namespace) { + dataSourceMetadata = new ExperimentDBService().loadMetadataFromDBByClusterName(dataSourceName, clusterName, internalVerbose); + } else { + internalVerbose = "true"; + dataSourceMetadata = new ExperimentDBService().loadMetadataFromDBByNamespace(dataSourceName, clusterName, namespace); + } + } + + if (null != dataSourceMetadata) { + dataSourceMetadataMap.put(dataSourceName, dataSourceMetadata); + } + } catch (Exception e) { + LOGGER.error("Loading saved Datasource metadata {} failed: {} ", dataSourceName, e.getMessage()); + } + + if (!dataSourceMetadataMap.containsKey(dataSourceName)) { + error = true; + sendErrorResponse( + response, + new Exception(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.MISSING_DATASOURCE_METADATA_EXCPTN), + HttpServletResponse.SC_BAD_REQUEST, + String.format(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.MISSING_DATASOURCE_METADATA_MSG, dataSourceName, clusterName, namespace) + ); + } + + } else { + error = true; + sendErrorResponse( + response, + new Exception(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_DATASOURCE_NAME_METADATA_EXCPTN), + HttpServletResponse.SC_BAD_REQUEST, + String.format(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_DATASOURCE_NAME_METADATA_MSG + , dataSourceName) + ); + } + + if (!error) { + // create Gson Object + Gson gsonObj = createGsonObject(); + gsonStr = gsonObj.toJson(dataSourceMetadataMap.get(dataSourceName)); + response.getWriter().println(gsonStr); + response.getWriter().close(); + statusValue = "success"; + } + } catch (Exception e) { + LOGGER.error("Exception: " + e.getMessage()); + e.printStackTrace(); + sendErrorResponse(response, e, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + } + } else { + sendErrorResponse( + response, + new Exception(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_QUERY_PARAM_VALUE), + HttpServletResponse.SC_BAD_REQUEST, + String.format(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_QUERY_PARAM_VALUE) + ); + } + } else { + sendErrorResponse( + response, + new Exception(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_QUERY_PARAM), + HttpServletResponse.SC_BAD_REQUEST, + String.format(AnalyzerErrorConstants.APIErrors.DSMetadataAPI.INVALID_QUERY_PARAM, invalidParams) + ); + } + } finally { + if (null != timerImportDSMetadata) { + MetricsConfig.timerImportDSMetadata = MetricsConfig.timerBImportDSMetadata.tag("status", statusValue).register(MetricsConfig.meterRegistry()); + timerImportDSMetadata.stop(MetricsConfig.timerImportDSMetadata); + } + } + } + + public void sendErrorResponse(HttpServletResponse response, Exception e, int httpStatusCode, String errorMsg) throws + IOException { + if (null != e) { + LOGGER.error(e.toString()); + e.printStackTrace(); + if (null == errorMsg) errorMsg = e.getMessage(); + } + response.sendError(httpStatusCode, errorMsg); + } + private Gson createGsonObject() { + return new GsonBuilder() + .disableHtmlEscaping() + .setPrettyPrinting() + .enableComplexMapKeySerialization() + .registerTypeAdapter(Date.class, new GsonUTCDateAdapter()) + .create(); + } + private boolean isValidBooleanValue(String value) { + return value != null && (value.equals("true") || value.equals("false")); + } +} diff --git a/src/main/java/com/autotune/analyzer/services/ListDatasources.java b/src/main/java/com/autotune/analyzer/services/ListDatasources.java new file mode 100644 index 000000000..1af77454d --- /dev/null +++ b/src/main/java/com/autotune/analyzer/services/ListDatasources.java @@ -0,0 +1,154 @@ +/******************************************************************************* + * Copyright (c) 2023 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +package com.autotune.analyzer.services; + +import com.autotune.analyzer.serviceObjects.ListDatasourcesAPIObject; +import com.autotune.analyzer.utils.AnalyzerConstants; +import com.autotune.analyzer.utils.AnalyzerErrorConstants; +import com.autotune.analyzer.utils.GsonUTCDateAdapter; +import com.autotune.common.datasource.DataSourceInfo; +import com.autotune.database.service.ExperimentDBService; +import com.autotune.utils.MetricsConfig; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.micrometer.core.instrument.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.CHARACTER_ENCODING; +import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.JSON_CONTENT_TYPE; + +/** + * Rest API used to list DataSources. + */ +@WebServlet(asyncSupported = true) +public class ListDatasources extends HttpServlet { + + private static final Logger LOGGER = LoggerFactory.getLogger(ListDatasources.class); + + @Override + public void init(ServletConfig config) throws ServletException { + super.init(config); + } + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String statusValue = "failure"; + Timer.Sample timerListDS = Timer.start(MetricsConfig.meterRegistry()); + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType(JSON_CONTENT_TYPE); + response.setCharacterEncoding(CHARACTER_ENCODING); + String gsonStr; + + String dataSourceName = request.getParameter(AnalyzerConstants.ServiceConstants.DATASOURCE_NAME); + + // Key = dataSourceName + HashMap dataSourceMap = new HashMap<>(); + boolean error = false; + + List dataSourceInfoList = new ArrayList<>(); + try { + + if (null != dataSourceName) { + try { + DataSourceInfo dataSource = new ExperimentDBService().loadDataSourceFromDBByName(dataSourceName); + if (null != dataSource) { + dataSourceMap.put(dataSourceName, dataSource); + } + } catch (Exception e) { + LOGGER.error("Loading saved Datasource {} failed: {} ", dataSourceName, e.getMessage()); + } + + if (!dataSourceMap.containsKey(dataSourceName)) { + error = true; + sendErrorResponse( + response, + new Exception(AnalyzerErrorConstants.APIErrors.ListDataSourcesAPI.INVALID_DATASOURCE_NAME_EXCPTN), + HttpServletResponse.SC_BAD_REQUEST, + String.format(AnalyzerErrorConstants.APIErrors.ListDataSourcesAPI.INVALID_DATASOURCE_NAME_MSG, dataSourceName) + ); + } + dataSourceInfoList.add(dataSourceMap.get(dataSourceName)); + } else { + try { + dataSourceInfoList = new ExperimentDBService().loadAllDataSources(); + } catch (Exception e) { + LOGGER.error("Loading saved Datasources failed: {} ", e.getMessage()); + } + } + + if (!error) { + try { + ListDatasourcesAPIObject listDatasourcesAPIObject = new ListDatasourcesAPIObject(); + listDatasourcesAPIObject.setDataSourceInfoList(dataSourceInfoList); + + // create Gson Object + Gson gsonObj = createGsonObject(); + gsonStr = gsonObj.toJson(listDatasourcesAPIObject); + response.getWriter().println(gsonStr); + response.getWriter().close(); + statusValue = "success"; + + } catch (Exception e) { + LOGGER.error(e.getMessage()); + } + } + + } catch (Exception e) { + LOGGER.error("Exception: " + e.getMessage()); + e.printStackTrace(); + sendErrorResponse(response, e, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + if (null != timerListDS) { + MetricsConfig.timerListDS = MetricsConfig.timerBListDS.tag("status", statusValue).register(MetricsConfig.meterRegistry()); + timerListDS.stop(MetricsConfig.timerListDS); + } + } + + } + + public void sendErrorResponse(HttpServletResponse response, Exception e, int httpStatusCode, String errorMsg) throws + IOException { + if (null != e) { + LOGGER.error(e.toString()); + e.printStackTrace(); + if (null == errorMsg) errorMsg = e.getMessage(); + } + response.sendError(httpStatusCode, errorMsg); + } + private Gson createGsonObject() { + return new GsonBuilder() + .disableHtmlEscaping() + .setPrettyPrinting() + .enableComplexMapKeySerialization() + .registerTypeAdapter(Date.class, new GsonUTCDateAdapter()) + .create(); + } + +} diff --git a/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java b/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java index 1b5a8ce31..a2cc6e1f7 100644 --- a/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java +++ b/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java @@ -310,6 +310,13 @@ public static final class ServiceConstants { public static final String LATEST = "latest"; public static final String EXPERIMENT_REGISTERED = "Registered successfully with Kruize! View registered experiments at /listExperiments"; public static final String RESULT_SAVED = "Results added successfully! View saved results at /listExperiments."; + public static final String DATASOURCE_NAME = "name"; + public static final String DATASOURCE = "datasource"; + public static final String DATASOURCE_PROVIDER = "provider"; + public static final String CLUSTER_NAME = "cluster_name"; + public static final String VERBOSE = "verbose"; + public static final String FALSE = "false"; + private ServiceConstants() { } diff --git a/src/main/java/com/autotune/analyzer/utils/AnalyzerErrorConstants.java b/src/main/java/com/autotune/analyzer/utils/AnalyzerErrorConstants.java index 1ae713e3d..a6feacbab 100644 --- a/src/main/java/com/autotune/analyzer/utils/AnalyzerErrorConstants.java +++ b/src/main/java/com/autotune/analyzer/utils/AnalyzerErrorConstants.java @@ -167,6 +167,27 @@ public static final class UpdateRecommendationsAPI { private UpdateRecommendationsAPI() { } } + public static final class ListDataSourcesAPI { + private ListDataSourcesAPI() { + + } + public static final String INVALID_DATASOURCE_NAME_EXCPTN = "Invalid Datasource Name"; + public static final String INVALID_DATASOURCE_NAME_MSG = "Given datasource name - \" %s \" either does not exist or is not valid"; + } + + public static final class DSMetadataAPI { + private DSMetadataAPI(){ + } + public static final String DATASOURCE_NAME_MANDATORY = KruizeConstants.JSONKeys.DATASOURCE + " is mandatory"; + public static final String INVALID_DATASOURCE_NAME_METADATA_EXCPTN = "Invalid DataSource Name"; + public static final String INVALID_DATASOURCE_NAME_METADATA_MSG = "Metadata for a given datasource name - \" %s \" either does not exist or is not valid"; + public static final String MISSING_DATASOURCE_METADATA_EXCPTN = "Invalid DataSource metadata"; + public static final String MISSING_DATASOURCE_METADATA_MSG = "Metadata for a given datasource - \" %s \", cluster name - \" %s \", namespace - \"%s \" " + + "either does not exist or is not valid"; + public static final String DATASOURCE_METADATA_IMPORT_ERROR_MSG = "Metadata cannot be imported for datasource - \" %s \" , either does not exist or is not valid"; + public static final String INVALID_QUERY_PARAM = "The query param(s) - \" %s \" is/are invalid"; + public static final String INVALID_QUERY_PARAM_VALUE = "The query param value(s) is/are invalid"; + } } public static final class ConversionErrors { diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index f06012983..da3133ae9 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -35,7 +35,6 @@ public void importMetadataFromDataSource(DataSourceInfo dataSourceInfo) { } String dataSourceName = dataSourceInfo.getName(); if(checkIfDataSourceMetadataExists(dataSourceName)) { - LOGGER.error("Metadata already exists for datasource: {}!", dataSourceName); return; } dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo); @@ -138,7 +137,7 @@ public void addMetadataToDB(DataSourceMetadataInfo dataSourceMetadataInfo) { // add the data source to DB addedToDB = new ExperimentDBService().addMetadataToDB(dataSourceMetadataInfo); if (addedToDB.isSuccess()) { - LOGGER.info("Metadata added to the DB successfully."); + LOGGER.debug("Metadata added to the DB successfully."); } else { LOGGER.error("Failed to add metadata to DB: {}", addedToDB.getMessage()); } diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAO.java b/src/main/java/com/autotune/database/dao/ExperimentDAO.java index fcd4fcc8e..ea1e8faa4 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAO.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAO.java @@ -79,8 +79,14 @@ public interface ExperimentDAO { // Load all the datasources List loadAllDataSources() throws Exception; - // Load data source cluster group by name - List loadMetadataByName(String clusterGroupName) throws Exception; + // Load data source metadata by datasource name + List loadMetadataByName(String dataSourceName) throws Exception; + + // Load data source metadata by cluster name + List loadMetadataByClusterName(String dataSourceName, String clusterName) throws Exception; + + // Load data source metadata by namespace + List loadMetadataByNamespace(String dataSourceName, String clusterName, String namespace) throws Exception; // add metadata ValidationOutputData addMetadataToDB(KruizeDSMetadataEntry kruizeDSMetadataEntry); diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java index 0b216b6e6..74474ffc2 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java @@ -826,6 +826,56 @@ public List loadMetadataByName(String dataSourceName) thr return kruizeMetadataList; } + /** + * Retrieves a list of KruizeDSMetadataEntry objects based on the specified datasource name and cluster name. + * + * @param dataSourceName The name of the datasource. + * @param clusterName The name of the cluster. + * @return A list of KruizeDSMetadataEntry objects associated with the provided datasource and cluster name. + * @throws Exception If there is an error while loading metadata from the database. + */ + @Override + public List loadMetadataByClusterName(String dataSourceName, String clusterName) throws Exception { + List kruizeMetadataList; + try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { + Query kruizeMetadataQuery = session.createQuery(SELECT_FROM_METADATA_BY_DATASOURCE_NAME_AND_CLUSTER_NAME, KruizeDSMetadataEntry.class) + .setParameter("datasource_name", dataSourceName) + .setParameter("cluster_name", clusterName); + + kruizeMetadataList = kruizeMetadataQuery.list(); + } catch (Exception e) { + LOGGER.error("Unable to load metadata with dataSourceName: {} and clusterName : {} : {}", dataSourceName, clusterName, e.getMessage()); + throw new Exception("Error while loading existing metadata object from database : " + e.getMessage()); + } + return kruizeMetadataList; + } + + /** + * Retrieves a list of KruizeDSMetadataEntry objects based on the specified + * datasource name, cluster name and namespace. + * + * @param dataSourceName The name of the datasource. + * @param clusterName The name of the cluster. + * @param namespace namespace + * @return A list of KruizeDSMetadataEntry objects associated with the provided datasource, cluster name and namespaces. + * @throws Exception If there is an error while loading metadata from the database. + */ + public List loadMetadataByNamespace(String dataSourceName, String clusterName, String namespace) throws Exception { + List kruizeMetadataList; + try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { + Query kruizeMetadataQuery = session.createQuery(SELECT_FROM_METADATA_BY_DATASOURCE_NAME_CLUSTER_NAME_AND_NAMESPACE, KruizeDSMetadataEntry.class) + .setParameter("datasource_name", dataSourceName) + .setParameter("cluster_name", clusterName) + .setParameter("namespace",namespace); + + kruizeMetadataList = kruizeMetadataQuery.list(); + } catch (Exception e) { + LOGGER.error("Unable to load metadata with dataSourceName: {}, clusterName : {} and namespace : {} : {}", dataSourceName, clusterName, namespace, e.getMessage()); + throw new Exception("Error while loading existing metadata object from database : " + e.getMessage()); + } + return kruizeMetadataList; + } + @Override public List loadMetadata() throws Exception { List kruizeMetadataList; diff --git a/src/main/java/com/autotune/database/helper/DBConstants.java b/src/main/java/com/autotune/database/helper/DBConstants.java index de69b667f..d9cd98ba2 100644 --- a/src/main/java/com/autotune/database/helper/DBConstants.java +++ b/src/main/java/com/autotune/database/helper/DBConstants.java @@ -13,6 +13,20 @@ public static final class SQLQUERY { public static final String SELECT_FROM_DATASOURCE_BY_NAME = "from KruizeDataSourceEntry kd WHERE kd.name = :name"; public static final String SELECT_FROM_METADATA = "from KruizeDSMetadataEntry"; public static final String SELECT_FROM_METADATA_BY_DATASOURCE_NAME = "from KruizeDSMetadataEntry km WHERE km.datasource_name = :dataSourceName"; + public static final String SELECT_FROM_METADATA_BY_DATASOURCE_NAME_AND_CLUSTER_NAME = + String.format("from KruizeDSMetadataEntry km " + + "WHERE km.datasource_name = :%s and " + + "km.cluster_name = :%s", + KruizeConstants.DataSourceConstants.DataSourceMetadataInfoJSONKeys.DATASOURCE_NAME, + KruizeConstants.DataSourceConstants.DataSourceMetadataInfoJSONKeys.CLUSTER_NAME); + public static final String SELECT_FROM_METADATA_BY_DATASOURCE_NAME_CLUSTER_NAME_AND_NAMESPACE = + String.format("from KruizeDSMetadataEntry km " + + "WHERE km.datasource_name = :%s and " + + "km.cluster_name = :%s and " + + "km.namespace = :%s", + KruizeConstants.DataSourceConstants.DataSourceMetadataInfoJSONKeys.DATASOURCE_NAME, + KruizeConstants.DataSourceConstants.DataSourceMetadataInfoJSONKeys.CLUSTER_NAME, + KruizeConstants.DataSourceConstants.DataSourceMetadataInfoJSONKeys.NAMESPACE); public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_DATE_RANGE_AND_LIMIT = String.format("from KruizeResultsEntry k " + "WHERE k.experiment_name = :%s and " + diff --git a/src/main/java/com/autotune/database/service/ExperimentDBService.java b/src/main/java/com/autotune/database/service/ExperimentDBService.java index 6588d4fec..c30cdc983 100644 --- a/src/main/java/com/autotune/database/service/ExperimentDBService.java +++ b/src/main/java/com/autotune/database/service/ExperimentDBService.java @@ -453,7 +453,7 @@ public DataSourceMetadataInfo loadMetadataFromDBByName(String dataSourceName, St List kruizeMetadataList = experimentDAO.loadMetadataByName(dataSourceName); List dataSourceDetailsInfoList = new ArrayList<>(); if (null != kruizeMetadataList && !kruizeMetadataList.isEmpty()) { - if (verbose.equals("false")) { + if (verbose.equals(AnalyzerConstants.ServiceConstants.FALSE)) { dataSourceDetailsInfoList = DBHelpers.Converters.KruizeObjectConverters .convertKruizeMetadataToClusterLevelDataSourceMetadata(kruizeMetadataList); } else { @@ -467,4 +467,49 @@ public DataSourceMetadataInfo loadMetadataFromDBByName(String dataSourceName, St return dataSourceDetailsInfoList.get(0); } + /** + * fetches metadata of specified datasource and cluster name from database + * @param dataSourceName String containing the name of datasource + * @param clusterName String containing the cluster name + * @param verbose + * @return DataSourceMetadataInfo object containing metadata + */ + public DataSourceMetadataInfo loadMetadataFromDBByClusterName(String dataSourceName, String clusterName, String verbose) throws Exception { + List kruizeMetadataList = experimentDAO.loadMetadataByClusterName(dataSourceName, clusterName); + List dataSourceMetadataInfoList = new ArrayList<>(); + if (null != kruizeMetadataList && !kruizeMetadataList.isEmpty()) { + if (verbose.equals(AnalyzerConstants.ServiceConstants.FALSE)) { + dataSourceMetadataInfoList = DBHelpers.Converters.KruizeObjectConverters + .convertKruizeMetadataToNamespaceLevelDataSourceMetadata(kruizeMetadataList); + } else { + dataSourceMetadataInfoList = DBHelpers.Converters.KruizeObjectConverters + .convertKruizeMetadataToDataSourceMetadataObject(kruizeMetadataList); + } + } + if (dataSourceMetadataInfoList.isEmpty()) + return null; + else + return dataSourceMetadataInfoList.get(0); + } + + /** + * fetches metadata of specified datasource,cluster and namespace from database + * @param dataSourceName String containing the name of datasource + * @param clusterName String containing the name of datasource + * @param namespace String containing the name of datasource + * @return DataSourceMetadataInfo object containing metadata + * @throws Exception + */ + public DataSourceMetadataInfo loadMetadataFromDBByNamespace(String dataSourceName, String clusterName, String namespace) throws Exception { + List kruizeMetadataList = experimentDAO.loadMetadataByNamespace(dataSourceName, clusterName, namespace); + List dataSourceMetadataInfoList = new ArrayList<>(); + if (null != kruizeMetadataList && !kruizeMetadataList.isEmpty()) { + dataSourceMetadataInfoList = DBHelpers.Converters.KruizeObjectConverters + .convertKruizeMetadataToDataSourceMetadataObject(kruizeMetadataList); + } + if (dataSourceMetadataInfoList.isEmpty()) + return null; + else + return dataSourceMetadataInfoList.get(0); + } } diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index 33a6ba8cc..3d111c24d 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -180,6 +180,10 @@ public static final class JSONKeys { public static final String CORES = "cores"; public static final String GIBIBYTE = "GiB"; + // Datasource JSON keys + public static final String DATASOURCES = "datasources"; + public static final String DATASOURCE_NAME = "datasource_name"; + // UI support JSON keys public static final String DATA = "data"; public static final String NAMESPACES = "namespaces"; diff --git a/src/main/java/com/autotune/utils/KruizeSupportedTypes.java b/src/main/java/com/autotune/utils/KruizeSupportedTypes.java index 28f3181a3..73091557f 100644 --- a/src/main/java/com/autotune/utils/KruizeSupportedTypes.java +++ b/src/main/java/com/autotune/utils/KruizeSupportedTypes.java @@ -76,6 +76,10 @@ private KruizeSupportedTypes() { } public static final Set KUBERNETES_OBJECTS_SUPPORTED = new HashSet<>(Arrays.asList("deployment", "pod", "container")); + + public static final Set DSMETADATA_QUERY_PARAMS_SUPPORTED = new HashSet<>(Arrays.asList( + "datasource", "cluster_name", "namespace", "verbose" + )); public static final Set SUPPORTED_FORMATS = new HashSet<>(Arrays.asList("cores", "m", "Bytes", "bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "kB", "KB", "MB", "GB", "TB", "PB", "EB", "K", "k", "M", "G", "T", "P", "E")); diff --git a/src/main/java/com/autotune/utils/MetricsConfig.java b/src/main/java/com/autotune/utils/MetricsConfig.java index 19ab3a43f..8baa24c5d 100644 --- a/src/main/java/com/autotune/utils/MetricsConfig.java +++ b/src/main/java/com/autotune/utils/MetricsConfig.java @@ -28,6 +28,9 @@ public class MetricsConfig { public String DB_METRIC_DESC = "Time taken for KruizeDB methods"; public static PrometheusMeterRegistry meterRegistry; + public static Timer timerListDS, timerImportDSMetadata; + public static Timer.Builder timerBListDS, timerBImportDSMetadata; + private MetricsConfig() { meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); meterRegistry.config().commonTags("application", "Kruize"); @@ -53,6 +56,9 @@ private MetricsConfig() { timerBLoadPerfProfileName = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method","loadPerformanceProfileByName"); timerBLoadAllPerfProfiles = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method","loadAllPerformanceProfiles"); + timerBListDS = Timer.builder("kruizeAPI").description(API_METRIC_DESC).tag("api","listDataSources").tag("method","GET"); + timerBImportDSMetadata = Timer.builder("kruizeAPI").description(API_METRIC_DESC).tag("api","importDataSourceMetadata").tag("method","POST"); + timerBImportDSMetadata = Timer.builder("kruizeAPI").description(API_METRIC_DESC).tag("api","importDataSourceMetadata").tag("method","GET"); new ClassLoaderMetrics().bindTo(meterRegistry); new ProcessorMetrics().bindTo(meterRegistry); new JvmGcMetrics().bindTo(meterRegistry); diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index 3ec751c48..7a3732517 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -63,6 +63,10 @@ public class ServerContext { public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS = ROOT_CONTEXT + "listTrialStatus"; public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS_END_POINT = EXPERIMENT_MANAGER_SERVER_URL + EXPERIMENT_MANAGER_LIST_TRIAL_STATUS; + //Datasource EndPoints + public static final String LIST_DATASOURCES = ROOT_CONTEXT + "datasources"; + public static final String DATASOURCE_METADATA = ROOT_CONTEXT + "dsmetadata"; + // UI support EndPoints public static final String QUERY_CONTEXT = ROOT_CONTEXT + "query/"; public static final String LIST_NAMESPACES = QUERY_CONTEXT + "listNamespaces";