From c37b0bc81e7a6a92aca193cec4f20ca8f716125c Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 26 Dec 2024 12:53:24 +0530 Subject: [PATCH 1/4] Add MetadataProfile database table --- migrations/kruize_local_ddl.sql | 1 + .../autotune/database/dao/ExperimentDAO.java | 4 + .../database/dao/ExperimentDAOImpl.java | 42 +++++++ .../autotune/database/helper/DBConstants.java | 2 + .../database/init/KruizeHibernateUtil.java | 2 + .../lm/KruizeLMMetadataProfileEntry.java | 107 ++++++++++++++++++ 6 files changed, 158 insertions(+) create mode 100644 src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java diff --git a/migrations/kruize_local_ddl.sql b/migrations/kruize_local_ddl.sql index 645dfbab1..0703b5467 100644 --- a/migrations/kruize_local_ddl.sql +++ b/migrations/kruize_local_ddl.sql @@ -6,3 +6,4 @@ alter table kruize_lm_experiments add column metadata_id bigint references krui alter table if exists kruize_lm_experiments add constraint UK_lm_experiment_name unique (experiment_name); create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name)); create table IF NOT EXISTS kruize_lm_recommendations (interval_end_time timestamp(6) not null, experiment_name varchar(255) not null, cluster_name varchar(255), extended_data jsonb, version varchar(255),experiment_type varchar(255), primary key (experiment_name, interval_end_time)) PARTITION BY RANGE (interval_end_time); +create table IF NOT EXISTS kruize_lm_metadata_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, query_variables jsonb, primary key (name)); \ No newline at end of file diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAO.java b/src/main/java/com/autotune/database/dao/ExperimentDAO.java index b8cc9c897..e2b965cb6 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAO.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAO.java @@ -6,6 +6,7 @@ import com.autotune.common.data.ValidationOutputData; import com.autotune.database.table.*; import com.autotune.database.table.lm.KruizeLMExperimentEntry; +import com.autotune.database.table.lm.KruizeLMMetadataProfileEntry; import com.autotune.database.table.lm.KruizeLMRecommendationEntry; import java.sql.Timestamp; @@ -65,6 +66,9 @@ public interface ExperimentDAO { // If Kruize restarts load all metric profiles List loadAllMetricProfiles() throws Exception; + // If Kruize restarts load all metadata profiles + List loadAllMetadataProfiles() throws Exception; + // Load a single experiment based on experimentName List loadExperimentByName(String experimentName) throws Exception; diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java index f4900e637..f0c31b2d5 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java @@ -25,6 +25,7 @@ import com.autotune.database.init.KruizeHibernateUtil; import com.autotune.database.table.*; import com.autotune.database.table.lm.KruizeLMExperimentEntry; +import com.autotune.database.table.lm.KruizeLMMetadataProfileEntry; import com.autotune.database.table.lm.KruizeLMRecommendationEntry; import com.autotune.utils.KruizeConstants; import com.autotune.utils.MetricsConfig; @@ -899,6 +900,26 @@ public List loadAllMetricProfiles() throws Exception { return entries; } + /** + * Fetches all the Metadata Profile records from KruizeLMMetadataProfileEntry database table + * + * @return List of all KruizeLMMetadataProfileEntry database objects + * @throws Exception + */ + @Override + public List loadAllMetadataProfiles() throws Exception { + String statusValue = "failure"; + + List entries = null; + try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { + entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_METADATA_PROFILE, KruizeLMMetadataProfileEntry.class).list(); + } catch (Exception e) { + LOGGER.error("Not able to load Metadata Profile due to {}", e.getMessage()); + throw new Exception("Error while loading existing Metadata Profile from database due to : " + e.getMessage()); + } + return entries; + } + @Override public List loadLMExperimentByName(String experimentName) throws Exception { //todo load only experimentStatus=inprogress , playback may not require completed experiments @@ -1209,6 +1230,27 @@ public List loadMetricProfileByName(String metricProfi return entries; } + /** + * Fetches Metadata Profile by name from KruizeLMMetadataProfileEntry database table + * + * @param metadataProfileName Metadata profile name + * @return List of KruizeLMMetadataProfileEntry objects + * @throws Exception + */ + public List loadMetadataProfileByName(String metadataProfileName) throws Exception { + String statusValue = "failure"; + Timer.Sample timerLoadMetadataProfileName = Timer.start(MetricsConfig.meterRegistry()); + List entries = null; + try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { + entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_METADATA_PROFILE_BY_NAME, KruizeLMMetadataProfileEntry.class) + .setParameter("name", metadataProfileName).list(); + } catch (Exception e) { + LOGGER.error("Not able to load Metadata Profile {} due to {}", metadataProfileName, e.getMessage()); + throw new Exception("Error while loading existing metadata profile from database due to : " + e.getMessage()); + } + return entries; + } + @Override public List getKruizeResultsEntry(String experiment_name, String cluster_name, Timestamp interval_start_time, Timestamp interval_end_time) throws Exception { List kruizeResultsEntryList = new ArrayList<>(); diff --git a/src/main/java/com/autotune/database/helper/DBConstants.java b/src/main/java/com/autotune/database/helper/DBConstants.java index fdcacc163..7abbb808a 100644 --- a/src/main/java/com/autotune/database/helper/DBConstants.java +++ b/src/main/java/com/autotune/database/helper/DBConstants.java @@ -68,6 +68,8 @@ public static final class SQLQUERY { public static final String SELECT_FROM_PERFORMANCE_PROFILE_BY_NAME = "from KruizePerformanceProfileEntry k WHERE k.name = :name"; public static final String SELECT_FROM_METRIC_PROFILE = "from KruizeMetricProfileEntry"; public static final String SELECT_FROM_METRIC_PROFILE_BY_NAME = "from KruizeMetricProfileEntry k WHERE k.name = :name"; + public static final String SELECT_FROM_METADATA_PROFILE = "from KruizeLMMetadataProfileEntry"; + public static final String SELECT_FROM_METADATA_PROFILE_BY_NAME = "from KruizeLMMetadataProfileEntry k WHERE k.name = :name"; public static final String DELETE_FROM_EXPERIMENTS_BY_EXP_NAME = "DELETE FROM KruizeExperimentEntry k WHERE k.experiment_name = :experimentName"; public static final String DELETE_FROM_RESULTS_BY_EXP_NAME = "DELETE FROM KruizeResultsEntry k WHERE k.experiment_name = :experimentName"; public static final String DELETE_FROM_RECOMMENDATIONS_BY_EXP_NAME = "DELETE FROM KruizeRecommendationEntry k WHERE k.experiment_name = :experimentName"; diff --git a/src/main/java/com/autotune/database/init/KruizeHibernateUtil.java b/src/main/java/com/autotune/database/init/KruizeHibernateUtil.java index 841ac99a9..37d9b70d3 100644 --- a/src/main/java/com/autotune/database/init/KruizeHibernateUtil.java +++ b/src/main/java/com/autotune/database/init/KruizeHibernateUtil.java @@ -18,6 +18,7 @@ import com.autotune.database.table.*; import com.autotune.database.table.lm.KruizeLMExperimentEntry; +import com.autotune.database.table.lm.KruizeLMMetadataProfileEntry; import com.autotune.database.table.lm.KruizeLMRecommendationEntry; import com.autotune.operator.KruizeDeploymentInfo; import org.hibernate.Session; @@ -65,6 +66,7 @@ public static void buildSessionFactory() { configuration.addAnnotatedClass(KruizeDSMetadataEntry.class); configuration.addAnnotatedClass(KruizeMetricProfileEntry.class); configuration.addAnnotatedClass(KruizeAuthenticationEntry.class); + configuration.addAnnotatedClass(KruizeLMMetadataProfileEntry.class); } LOGGER.info("DB is trying to connect to {}", connectionURL); sfTemp = configuration.buildSessionFactory(); diff --git a/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java b/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java new file mode 100644 index 000000000..e78b0b08a --- /dev/null +++ b/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java @@ -0,0 +1,107 @@ +/******************************************************************************* + * Copyright (c) 2024 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.database.table.lm; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +/** + * This is a Java class named KruizeLMMetadataProfileEntry annotated with JPA annotations. + * It represents a table named kruize_lm_metadata_profiles in a relational database. + *

+ * The class has the following fields: + *

+ * id: A unique identifier for each metadata profile detail. + * apiVersion: A string representing version of the Kubernetes API to create this object + * kind: A string representing type of kubernetes object + * metadata: A JSON object containing the metadata of the CRD, including name field + * name: A string representing the name of the metadata profile. + * profile_version: A string representing the version of the metadata profile. + * k8s_type: A string representing kubernetes type. + * query_variables: A JSON object containing metadata queries + */ +@Entity +@Table(name = "kruize_lm_metadata_profiles") +public class KruizeLMMetadataProfileEntry { + private String api_version; + private String kind; + @JdbcTypeCode(SqlTypes.JSON) + private JsonNode metadata; + @Id + private String name; + private double profile_version; + private String k8s_type; + @JdbcTypeCode(SqlTypes.JSON) + private JsonNode query_variables; + + public String getApi_version() { + return api_version; + } + + public void setApi_version(String api_version) { + this.api_version = api_version; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public JsonNode getMetadata() { + return metadata; + } + + public void setMetadata(JsonNode metadata) { + this.metadata = metadata; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getProfile_version() { + return profile_version; + } + + public void setProfile_version(double profile_version) { + this.profile_version = profile_version; + } + + public String getK8s_type() { + return k8s_type; + } + + public void setK8s_type(String k8s_type) { + this.k8s_type = k8s_type; + } + + public JsonNode getQuery_variables() { + return query_variables; + } + + public void setQuery_variables(JsonNode query_variables) { + this.query_variables = query_variables; + } +} \ No newline at end of file From c54a6311c038233600181592b2253236fefe1614 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 26 Dec 2024 13:42:49 +0530 Subject: [PATCH 2/4] Add missing EOF --- migrations/kruize_local_ddl.sql | 2 +- .../database/table/lm/KruizeLMMetadataProfileEntry.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/kruize_local_ddl.sql b/migrations/kruize_local_ddl.sql index 0703b5467..7a8b32054 100644 --- a/migrations/kruize_local_ddl.sql +++ b/migrations/kruize_local_ddl.sql @@ -6,4 +6,4 @@ alter table kruize_lm_experiments add column metadata_id bigint references krui alter table if exists kruize_lm_experiments add constraint UK_lm_experiment_name unique (experiment_name); create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name)); create table IF NOT EXISTS kruize_lm_recommendations (interval_end_time timestamp(6) not null, experiment_name varchar(255) not null, cluster_name varchar(255), extended_data jsonb, version varchar(255),experiment_type varchar(255), primary key (experiment_name, interval_end_time)) PARTITION BY RANGE (interval_end_time); -create table IF NOT EXISTS kruize_lm_metadata_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, query_variables jsonb, primary key (name)); \ No newline at end of file +create table IF NOT EXISTS kruize_lm_metadata_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, query_variables jsonb, primary key (name)); diff --git a/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java b/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java index e78b0b08a..dba97038c 100644 --- a/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java +++ b/src/main/java/com/autotune/database/table/lm/KruizeLMMetadataProfileEntry.java @@ -104,4 +104,4 @@ public JsonNode getQuery_variables() { public void setQuery_variables(JsonNode query_variables) { this.query_variables = query_variables; } -} \ No newline at end of file +} From da321709c5e4c79851dc2ea38914b67fab298553 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 26 Dec 2024 18:03:51 +0530 Subject: [PATCH 3/4] Add DB functions --- .../autotune/database/dao/ExperimentDAO.java | 6 ++++ .../database/dao/ExperimentDAOImpl.java | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAO.java b/src/main/java/com/autotune/database/dao/ExperimentDAO.java index e2b965cb6..7addef21d 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAO.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAO.java @@ -37,6 +37,9 @@ public interface ExperimentDAO { // Add Metric Profile to DB public ValidationOutputData addMetricProfileToDB(KruizeMetricProfileEntry kruizeMetricProfileEntry); + // Add Metadata Profile to DB + public ValidationOutputData addMetadataProfileToDB(KruizeLMMetadataProfileEntry kruizeMetadataProfileEntry); + // Add DataSource to DB ValidationOutputData addDataSourceToDB(KruizeDataSourceEntry kruizeDataSourceEntry, ValidationOutputData validationOutputData); @@ -95,6 +98,9 @@ public interface ExperimentDAO { // Load a single Metric Profile based on name List loadMetricProfileByName(String metricProfileName) throws Exception; + // Load a single Metadata Profile based on name + List loadMetadataProfileByName(String metadataProfileName) throws Exception; + // Delete metric profile for the specified metric profile name public ValidationOutputData deleteKruizeMetricProfileEntryByName(String metricProfileName); diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java index f0c31b2d5..b191133e7 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java @@ -523,6 +523,36 @@ public ValidationOutputData addMetricProfileToDB(KruizeMetricProfileEntry kruize return validationOutputData; } + /** + * Add MetadataProfile to database + * + * @param kruizeMetadataProfileEntry Metadata Profile Database object to be added + * @return validationOutputData contains the status of the DB insert operation + */ + @Override + public ValidationOutputData addMetadataProfileToDB(KruizeLMMetadataProfileEntry kruizeMetadataProfileEntry) { + ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null); + Transaction tx = null; + try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { + try { + tx = session.beginTransaction(); + session.persist(kruizeMetadataProfileEntry); + tx.commit(); + validationOutputData.setSuccess(true); + } catch (HibernateException e) { + LOGGER.error("Not able to save metadata profile due to {}", e.getMessage()); + if (tx != null) tx.rollback(); + e.printStackTrace(); + validationOutputData.setSuccess(false); + validationOutputData.setMessage(e.getMessage()); + } + } catch (Exception e) { + LOGGER.error("Not able to save metadata profile source due to {}", e.getMessage()); + validationOutputData.setMessage(e.getMessage()); + } + return validationOutputData; + } + /** * @param kruizeDataSourceEntry * @param validationOutputData From dfa4bb462807d46dd8a5c785fdd7589983051639 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 23 Jan 2025 13:21:12 +0530 Subject: [PATCH 4/4] Track execution time for database operations --- .../database/dao/ExperimentDAOImpl.java | 19 +++++++++++++++++++ .../com/autotune/utils/MetricsConfig.java | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java index b191133e7..794be000a 100644 --- a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java +++ b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java @@ -532,6 +532,8 @@ public ValidationOutputData addMetricProfileToDB(KruizeMetricProfileEntry kruize @Override public ValidationOutputData addMetadataProfileToDB(KruizeLMMetadataProfileEntry kruizeMetadataProfileEntry) { ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null); + String statusValue = "failure"; + Timer.Sample timerAddMetadataProfileDB = Timer.start(MetricsConfig.meterRegistry()); Transaction tx = null; try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { try { @@ -549,7 +551,13 @@ public ValidationOutputData addMetadataProfileToDB(KruizeLMMetadataProfileEntry } catch (Exception e) { LOGGER.error("Not able to save metadata profile source due to {}", e.getMessage()); validationOutputData.setMessage(e.getMessage()); + } finally { + if (null != timerAddMetadataProfileDB) { + MetricsConfig.timerAddMetadataProfileDB = MetricsConfig.timerBAddMetadataProfileDB.tag("status", statusValue).register(MetricsConfig.meterRegistry()); + timerAddMetadataProfileDB.stop(MetricsConfig.timerAddMetadataProfileDB); + } } + return validationOutputData; } @@ -939,6 +947,7 @@ public List loadAllMetricProfiles() throws Exception { @Override public List loadAllMetadataProfiles() throws Exception { String statusValue = "failure"; + Timer.Sample timerLoadAllMetadataProfiles = Timer.start(MetricsConfig.meterRegistry()); List entries = null; try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) { @@ -946,6 +955,11 @@ public List loadAllMetadataProfiles() throws Excep } catch (Exception e) { LOGGER.error("Not able to load Metadata Profile due to {}", e.getMessage()); throw new Exception("Error while loading existing Metadata Profile from database due to : " + e.getMessage()); + } finally { + if (null != timerLoadAllMetadataProfiles) { + MetricsConfig.timerLoadAllMetadataProfiles = MetricsConfig.timerBLoadAllMetadataProfiles.tag("status", statusValue).register(MetricsConfig.meterRegistry()); + timerLoadAllMetadataProfiles.stop(MetricsConfig.timerLoadAllMetadataProfiles); + } } return entries; } @@ -1277,6 +1291,11 @@ public List loadMetadataProfileByName(String metad } catch (Exception e) { LOGGER.error("Not able to load Metadata Profile {} due to {}", metadataProfileName, e.getMessage()); throw new Exception("Error while loading existing metadata profile from database due to : " + e.getMessage()); + } finally { + if (null != timerLoadMetadataProfileName) { + MetricsConfig.timerLoadMetadataProfileName = MetricsConfig.timerBLoadMetadataProfileName.tag("status", statusValue).register(MetricsConfig.meterRegistry()); + timerLoadMetadataProfileName.stop(MetricsConfig.timerLoadMetadataProfileName); + } } return entries; } diff --git a/src/main/java/com/autotune/utils/MetricsConfig.java b/src/main/java/com/autotune/utils/MetricsConfig.java index b7afa12f7..12b761055 100644 --- a/src/main/java/com/autotune/utils/MetricsConfig.java +++ b/src/main/java/com/autotune/utils/MetricsConfig.java @@ -21,6 +21,7 @@ public class MetricsConfig { public static Timer timerLoadAllRec, timerLoadAllExp, timerLoadAllResults; public static Timer timerAddRecDB, timerAddResultsDB, timerAddExpDB, timerAddBulkResultsDB; public static Timer timerAddPerfProfileDB, timerLoadPerfProfileName, timerLoadAllPerfProfiles; + public static Timer timerAddMetadataProfileDB, timerLoadMetadataProfileName, timerLoadAllMetadataProfiles; public static Timer timerImportMetadata, timerGetMetadata; public static Timer timerJobStatus, timerCreateBulkJob, timerGetExpMap, timerCreateBulkExp, timerGenerateBulkRec, timerRunJob; public static Counter timerKruizeNotifications , timerBulkJobs; @@ -35,6 +36,7 @@ public class MetricsConfig { public static Timer.Builder timerBListDS, timerBImportDSMetadata, timerBListDSMetadata; public static Timer.Builder timerBImportMetadata, timerBGetMetadata; public static Timer.Builder timerBJobStatus, timerBCreateBulkJob, timerBGetExpMap, timerBCreateBulkExp, timerBGenerateBulkRec, timerBRunJob; + public static Timer.Builder timerBAddMetadataProfileDB, timerBLoadMetadataProfileName, timerBLoadAllMetadataProfiles; private static MetricsConfig INSTANCE; public String API_METRIC_DESC = "Time taken for Kruize APIs"; public String DB_METRIC_DESC = "Time taken for KruizeDB methods"; @@ -82,6 +84,10 @@ private MetricsConfig() { timerBBulkRunJobs = Gauge.builder("kruizeAPI_active_jobs_count", activeJobs, AtomicInteger::get).description("No.of bulk jobs running").tags("api", "bulk", "method", "runBulkJob" , "status", "running"); timerBBulkRunJobs.register(meterRegistry); + timerBAddMetadataProfileDB = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method", "addMetadataProfileToDB"); + timerBLoadMetadataProfileName = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method", "loadMetadataProfileByName"); + timerBLoadAllMetadataProfiles = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method", "loadAllMetadataProfiles"); + new ClassLoaderMetrics().bindTo(meterRegistry); new ProcessorMetrics().bindTo(meterRegistry); new JvmGcMetrics().bindTo(meterRegistry);