Skip to content

Commit

Permalink
Merge pull request #1441 from shreyabiradar07/metadata-profile-db
Browse files Browse the repository at this point in the history
3. Add MetadataProfile database
  • Loading branch information
dinogun authored Jan 24, 2025
2 parents 941e748 + dfa4bb4 commit 6c48c41
Show file tree
Hide file tree
Showing 7 changed files with 219 additions and 0 deletions.
1 change: 1 addition & 0 deletions migrations/kruize_local_ddl.sql
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ alter table kruize_lm_experiments add column metadata_id bigint references krui
alter table if exists kruize_lm_experiments add constraint UK_lm_experiment_name unique (experiment_name);
create table IF NOT EXISTS kruize_metric_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, slo jsonb, primary key (name));
create table IF NOT EXISTS kruize_lm_recommendations (interval_end_time timestamp(6) not null, experiment_name varchar(255) not null, cluster_name varchar(255), extended_data jsonb, version varchar(255),experiment_type varchar(255), primary key (experiment_name, interval_end_time)) PARTITION BY RANGE (interval_end_time);
create table IF NOT EXISTS kruize_lm_metadata_profiles (api_version varchar(255), kind varchar(255), metadata jsonb, name varchar(255) not null, k8s_type varchar(255), profile_version float(53) not null, query_variables jsonb, primary key (name));
10 changes: 10 additions & 0 deletions src/main/java/com/autotune/database/dao/ExperimentDAO.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.autotune.common.data.ValidationOutputData;
import com.autotune.database.table.*;
import com.autotune.database.table.lm.KruizeLMExperimentEntry;
import com.autotune.database.table.lm.KruizeLMMetadataProfileEntry;
import com.autotune.database.table.lm.KruizeLMRecommendationEntry;

import java.sql.Timestamp;
Expand Down Expand Up @@ -36,6 +37,9 @@ public interface ExperimentDAO {
// Add Metric Profile to DB
public ValidationOutputData addMetricProfileToDB(KruizeMetricProfileEntry kruizeMetricProfileEntry);

// Add Metadata Profile to DB
public ValidationOutputData addMetadataProfileToDB(KruizeLMMetadataProfileEntry kruizeMetadataProfileEntry);

// Add DataSource to DB
ValidationOutputData addDataSourceToDB(KruizeDataSourceEntry kruizeDataSourceEntry, ValidationOutputData validationOutputData);

Expand Down Expand Up @@ -65,6 +69,9 @@ public interface ExperimentDAO {
// If Kruize restarts load all metric profiles
List<KruizeMetricProfileEntry> loadAllMetricProfiles() throws Exception;

// If Kruize restarts load all metadata profiles
List<KruizeLMMetadataProfileEntry> loadAllMetadataProfiles() throws Exception;

// Load a single experiment based on experimentName
List<KruizeExperimentEntry> loadExperimentByName(String experimentName) throws Exception;

Expand All @@ -91,6 +98,9 @@ public interface ExperimentDAO {
// Load a single Metric Profile based on name
List<KruizeMetricProfileEntry> loadMetricProfileByName(String metricProfileName) throws Exception;

// Load a single Metadata Profile based on name
List<KruizeLMMetadataProfileEntry> loadMetadataProfileByName(String metadataProfileName) throws Exception;

// Delete metric profile for the specified metric profile name
public ValidationOutputData deleteKruizeMetricProfileEntryByName(String metricProfileName);

Expand Down
91 changes: 91 additions & 0 deletions src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.autotune.database.init.KruizeHibernateUtil;
import com.autotune.database.table.*;
import com.autotune.database.table.lm.KruizeLMExperimentEntry;
import com.autotune.database.table.lm.KruizeLMMetadataProfileEntry;
import com.autotune.database.table.lm.KruizeLMRecommendationEntry;
import com.autotune.utils.KruizeConstants;
import com.autotune.utils.MetricsConfig;
Expand Down Expand Up @@ -522,6 +523,44 @@ public ValidationOutputData addMetricProfileToDB(KruizeMetricProfileEntry kruize
return validationOutputData;
}

/**
* Add MetadataProfile to database
*
* @param kruizeMetadataProfileEntry Metadata Profile Database object to be added
* @return validationOutputData contains the status of the DB insert operation
*/
@Override
public ValidationOutputData addMetadataProfileToDB(KruizeLMMetadataProfileEntry kruizeMetadataProfileEntry) {
ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null);
String statusValue = "failure";
Timer.Sample timerAddMetadataProfileDB = Timer.start(MetricsConfig.meterRegistry());
Transaction tx = null;
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
try {
tx = session.beginTransaction();
session.persist(kruizeMetadataProfileEntry);
tx.commit();
validationOutputData.setSuccess(true);
} catch (HibernateException e) {
LOGGER.error("Not able to save metadata profile due to {}", e.getMessage());
if (tx != null) tx.rollback();
e.printStackTrace();
validationOutputData.setSuccess(false);
validationOutputData.setMessage(e.getMessage());
}
} catch (Exception e) {
LOGGER.error("Not able to save metadata profile source due to {}", e.getMessage());
validationOutputData.setMessage(e.getMessage());
} finally {
if (null != timerAddMetadataProfileDB) {
MetricsConfig.timerAddMetadataProfileDB = MetricsConfig.timerBAddMetadataProfileDB.tag("status", statusValue).register(MetricsConfig.meterRegistry());
timerAddMetadataProfileDB.stop(MetricsConfig.timerAddMetadataProfileDB);
}
}

return validationOutputData;
}

/**
* @param kruizeDataSourceEntry
* @param validationOutputData
Expand Down Expand Up @@ -899,6 +938,32 @@ public List<KruizeMetricProfileEntry> loadAllMetricProfiles() throws Exception {
return entries;
}

/**
* Fetches all the Metadata Profile records from KruizeLMMetadataProfileEntry database table
*
* @return List of all KruizeLMMetadataProfileEntry database objects
* @throws Exception
*/
@Override
public List<KruizeLMMetadataProfileEntry> loadAllMetadataProfiles() throws Exception {
String statusValue = "failure";
Timer.Sample timerLoadAllMetadataProfiles = Timer.start(MetricsConfig.meterRegistry());

List<KruizeLMMetadataProfileEntry> entries = null;
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_METADATA_PROFILE, KruizeLMMetadataProfileEntry.class).list();
} catch (Exception e) {
LOGGER.error("Not able to load Metadata Profile due to {}", e.getMessage());
throw new Exception("Error while loading existing Metadata Profile from database due to : " + e.getMessage());
} finally {
if (null != timerLoadAllMetadataProfiles) {
MetricsConfig.timerLoadAllMetadataProfiles = MetricsConfig.timerBLoadAllMetadataProfiles.tag("status", statusValue).register(MetricsConfig.meterRegistry());
timerLoadAllMetadataProfiles.stop(MetricsConfig.timerLoadAllMetadataProfiles);
}
}
return entries;
}

@Override
public List<KruizeLMExperimentEntry> loadLMExperimentByName(String experimentName) throws Exception {
//todo load only experimentStatus=inprogress , playback may not require completed experiments
Expand Down Expand Up @@ -1209,6 +1274,32 @@ public List<KruizeMetricProfileEntry> loadMetricProfileByName(String metricProfi
return entries;
}

/**
* Fetches Metadata Profile by name from KruizeLMMetadataProfileEntry database table
*
* @param metadataProfileName Metadata profile name
* @return List of KruizeLMMetadataProfileEntry objects
* @throws Exception
*/
public List<KruizeLMMetadataProfileEntry> loadMetadataProfileByName(String metadataProfileName) throws Exception {
String statusValue = "failure";
Timer.Sample timerLoadMetadataProfileName = Timer.start(MetricsConfig.meterRegistry());
List<KruizeLMMetadataProfileEntry> entries = null;
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_METADATA_PROFILE_BY_NAME, KruizeLMMetadataProfileEntry.class)
.setParameter("name", metadataProfileName).list();
} catch (Exception e) {
LOGGER.error("Not able to load Metadata Profile {} due to {}", metadataProfileName, e.getMessage());
throw new Exception("Error while loading existing metadata profile from database due to : " + e.getMessage());
} finally {
if (null != timerLoadMetadataProfileName) {
MetricsConfig.timerLoadMetadataProfileName = MetricsConfig.timerBLoadMetadataProfileName.tag("status", statusValue).register(MetricsConfig.meterRegistry());
timerLoadMetadataProfileName.stop(MetricsConfig.timerLoadMetadataProfileName);
}
}
return entries;
}

@Override
public List<KruizeResultsEntry> getKruizeResultsEntry(String experiment_name, String cluster_name, Timestamp interval_start_time, Timestamp interval_end_time) throws Exception {
List<KruizeResultsEntry> kruizeResultsEntryList = new ArrayList<>();
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/autotune/database/helper/DBConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ public static final class SQLQUERY {
public static final String SELECT_FROM_PERFORMANCE_PROFILE_BY_NAME = "from KruizePerformanceProfileEntry k WHERE k.name = :name";
public static final String SELECT_FROM_METRIC_PROFILE = "from KruizeMetricProfileEntry";
public static final String SELECT_FROM_METRIC_PROFILE_BY_NAME = "from KruizeMetricProfileEntry k WHERE k.name = :name";
public static final String SELECT_FROM_METADATA_PROFILE = "from KruizeLMMetadataProfileEntry";
public static final String SELECT_FROM_METADATA_PROFILE_BY_NAME = "from KruizeLMMetadataProfileEntry k WHERE k.name = :name";
public static final String DELETE_FROM_EXPERIMENTS_BY_EXP_NAME = "DELETE FROM KruizeExperimentEntry k WHERE k.experiment_name = :experimentName";
public static final String DELETE_FROM_RESULTS_BY_EXP_NAME = "DELETE FROM KruizeResultsEntry k WHERE k.experiment_name = :experimentName";
public static final String DELETE_FROM_RECOMMENDATIONS_BY_EXP_NAME = "DELETE FROM KruizeRecommendationEntry k WHERE k.experiment_name = :experimentName";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import com.autotune.database.table.*;
import com.autotune.database.table.lm.KruizeLMExperimentEntry;
import com.autotune.database.table.lm.KruizeLMMetadataProfileEntry;
import com.autotune.database.table.lm.KruizeLMRecommendationEntry;
import com.autotune.operator.KruizeDeploymentInfo;
import org.hibernate.Session;
Expand Down Expand Up @@ -65,6 +66,7 @@ public static void buildSessionFactory() {
configuration.addAnnotatedClass(KruizeDSMetadataEntry.class);
configuration.addAnnotatedClass(KruizeMetricProfileEntry.class);
configuration.addAnnotatedClass(KruizeAuthenticationEntry.class);
configuration.addAnnotatedClass(KruizeLMMetadataProfileEntry.class);
}
LOGGER.info("DB is trying to connect to {}", connectionURL);
sfTemp = configuration.buildSessionFactory();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*******************************************************************************
* Copyright (c) 2024 Red Hat, IBM Corporation and others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
package com.autotune.database.table.lm;

import com.fasterxml.jackson.databind.JsonNode;
import jakarta.persistence.*;
import org.hibernate.annotations.JdbcTypeCode;
import org.hibernate.type.SqlTypes;

/**
* This is a Java class named KruizeLMMetadataProfileEntry annotated with JPA annotations.
* It represents a table named kruize_lm_metadata_profiles in a relational database.
* <p>
* The class has the following fields:
* <p>
* id: A unique identifier for each metadata profile detail.
* apiVersion: A string representing version of the Kubernetes API to create this object
* kind: A string representing type of kubernetes object
* metadata: A JSON object containing the metadata of the CRD, including name field
* name: A string representing the name of the metadata profile.
* profile_version: A string representing the version of the metadata profile.
* k8s_type: A string representing kubernetes type.
* query_variables: A JSON object containing metadata queries
*/
@Entity
@Table(name = "kruize_lm_metadata_profiles")
public class KruizeLMMetadataProfileEntry {
private String api_version;
private String kind;
@JdbcTypeCode(SqlTypes.JSON)
private JsonNode metadata;
@Id
private String name;
private double profile_version;
private String k8s_type;
@JdbcTypeCode(SqlTypes.JSON)
private JsonNode query_variables;

public String getApi_version() {
return api_version;
}

public void setApi_version(String api_version) {
this.api_version = api_version;
}

public String getKind() {
return kind;
}

public void setKind(String kind) {
this.kind = kind;
}

public JsonNode getMetadata() {
return metadata;
}

public void setMetadata(JsonNode metadata) {
this.metadata = metadata;
}

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public double getProfile_version() {
return profile_version;
}

public void setProfile_version(double profile_version) {
this.profile_version = profile_version;
}

public String getK8s_type() {
return k8s_type;
}

public void setK8s_type(String k8s_type) {
this.k8s_type = k8s_type;
}

public JsonNode getQuery_variables() {
return query_variables;
}

public void setQuery_variables(JsonNode query_variables) {
this.query_variables = query_variables;
}
}
6 changes: 6 additions & 0 deletions src/main/java/com/autotune/utils/MetricsConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class MetricsConfig {
public static Timer timerLoadAllRec, timerLoadAllExp, timerLoadAllResults;
public static Timer timerAddRecDB, timerAddResultsDB, timerAddExpDB, timerAddBulkResultsDB;
public static Timer timerAddPerfProfileDB, timerLoadPerfProfileName, timerLoadAllPerfProfiles;
public static Timer timerAddMetadataProfileDB, timerLoadMetadataProfileName, timerLoadAllMetadataProfiles;
public static Timer timerImportMetadata, timerGetMetadata;
public static Timer timerJobStatus, timerCreateBulkJob, timerGetExpMap, timerCreateBulkExp, timerGenerateBulkRec, timerRunJob;
public static Counter timerKruizeNotifications , timerBulkJobs;
Expand All @@ -35,6 +36,7 @@ public class MetricsConfig {
public static Timer.Builder timerBListDS, timerBImportDSMetadata, timerBListDSMetadata;
public static Timer.Builder timerBImportMetadata, timerBGetMetadata;
public static Timer.Builder timerBJobStatus, timerBCreateBulkJob, timerBGetExpMap, timerBCreateBulkExp, timerBGenerateBulkRec, timerBRunJob;
public static Timer.Builder timerBAddMetadataProfileDB, timerBLoadMetadataProfileName, timerBLoadAllMetadataProfiles;
private static MetricsConfig INSTANCE;
public String API_METRIC_DESC = "Time taken for Kruize APIs";
public String DB_METRIC_DESC = "Time taken for KruizeDB methods";
Expand Down Expand Up @@ -82,6 +84,10 @@ private MetricsConfig() {
timerBBulkRunJobs = Gauge.builder("kruizeAPI_active_jobs_count", activeJobs, AtomicInteger::get).description("No.of bulk jobs running").tags("api", "bulk", "method", "runBulkJob" , "status", "running");
timerBBulkRunJobs.register(meterRegistry);

timerBAddMetadataProfileDB = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method", "addMetadataProfileToDB");
timerBLoadMetadataProfileName = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method", "loadMetadataProfileByName");
timerBLoadAllMetadataProfiles = Timer.builder("kruizeDB").description(DB_METRIC_DESC).tag("method", "loadAllMetadataProfiles");

new ClassLoaderMetrics().bindTo(meterRegistry);
new ProcessorMetrics().bindTo(meterRegistry);
new JvmGcMetrics().bindTo(meterRegistry);
Expand Down

0 comments on commit 6c48c41

Please sign in to comment.