Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Adds cluster_name parameter to listRecommendations API #944

Closed
wants to merge 10 commits into from
Closed

[WIP] Adds cluster_name parameter to listRecommendations API #944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3b5499c
add declaration for clusterName
shreyabiradar07 Sep 14, 2023
e9b4117
rebase with mvp_demo adding clusterName functionality
shreyabiradar07 Sep 14, 2023
8dc5622
resolve conflicts of ExperimentDAOImpl and DBConstants
shreyabiradar07 Sep 14, 2023
47b0236
resolve code conflicts by loading results by clusterName
shreyabiradar07 Sep 15, 2023
9632fe6
refactor dbconstants for cluster_name
shreyabiradar07 Sep 29, 2023
4d6ff95
refactor code changes with experiment_name as key
shreyabiradar07 Oct 3, 2023
1bf0002
insert spaces after braces and parentheses
shreyabiradar07 Oct 4, 2023
9a0f06b
removes redundant methods and DB constants using cluster_name to load…
shreyabiradar07 Oct 4, 2023
117fa52
refactor error handling and logger messages
shreyabiradar07 Oct 4, 2023
962283c
add DB index for cluster_name in kruize_recommendations table
shreyabiradar07 Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions migrations/kruize_experiments_ddl.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@ create table IF NOT EXISTS kruize_results (interval_start_time timestamp(6) not
alter table if exists kruize_experiments add constraint UK_experiment_name unique (experiment_name);
create index IF NOT EXISTS idx_recommendation_experiment_name on kruize_recommendations (experiment_name);
create index IF NOT EXISTS idx_recommendation_interval_end_time on kruize_recommendations (interval_end_time);
create index IF NOT EXISTS idx_recommendation_cluster_name on kruize_recommendations (cluster_name);
create index IF NOT EXISTS idx_result_experiment_name on kruize_results (experiment_name);
create index IF NOT EXISTS idx_result_interval_end_time on kruize_results (interval_end_time);
54 changes: 54 additions & 0 deletions src/main/java/com/autotune/analyzer/services/ListRecommendations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
// Set the character encoding of the request to UTF-8
request.setCharacterEncoding(CHARACTER_ENCODING);
String experimentName = request.getParameter(AnalyzerConstants.ServiceConstants.EXPERIMENT_NAME);
String clusterName = request.getParameter(KruizeConstants.JSONKeys.CLUSTER_NAME);
String latestRecommendation = request.getParameter(AnalyzerConstants.ServiceConstants.LATEST);
String monitoringEndTime = request.getParameter(KruizeConstants.JSONKeys.MONITORING_END_TIME);
Timestamp monitoringEndTimestamp = null;
Expand Down Expand Up @@ -146,6 +147,59 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
String.format(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_EXPERIMENT_NAME_MSG, experimentName)
);
}
} else if (null != clusterName) {
try {
new ExperimentDBService().loadExperimentsAndRecommendationsFromDBByClusterName(mKruizeExperimentMap, clusterName);
} catch (Exception e) {
LOGGER.error("Loading experiments and recommendations based on cluster {} failed: {} ", clusterName, e.getMessage());
}
// Check if cluster exists
if (!mKruizeExperimentMap.isEmpty()) {
// Check if timestamp is passed
if (null != monitoringEndTime && !monitoringEndTime.isEmpty()) {
monitoringEndTime = monitoringEndTime.trim();
if (Utils.DateUtils.isAValidDate(KruizeConstants.DateFormats.STANDARD_JSON_DATE_FORMAT, monitoringEndTime)) {
Date mEndTime = Utils.DateUtils.getDateFrom(KruizeConstants.DateFormats.STANDARD_JSON_DATE_FORMAT, monitoringEndTime);
monitoringEndTimestamp = new Timestamp(mEndTime.getTime());
// Check if timestamp exists in recommendations
for (String expName : mKruizeExperimentMap.keySet()) {
boolean timestampExists = ServiceHelpers.KruizeObjectOperations.checkRecommendationTimestampExists(mKruizeExperimentMap.get(expName), monitoringEndTime);
if (timestampExists) {
kruizeObjectList.add(mKruizeExperimentMap.get(expName));
checkForTimestamp = true;
}
}
if (kruizeObjectList.isEmpty()) {
error = true;
sendErrorResponse(
response,
new Exception(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.RECOMMENDATION_DOES_NOT_EXIST_EXCPTN),
HttpServletResponse.SC_BAD_REQUEST,
String.format(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.RECOMMENDATION_DOES_NOT_EXIST_MSG, monitoringEndTime)
);
}
} else {
error = true;
sendErrorResponse(
response,
new Exception(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_TIMESTAMP_EXCPTN),
HttpServletResponse.SC_BAD_REQUEST,
String.format(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_TIMESTAMP_MSG, monitoringEndTime)
);
}
} else {
// Add all experiments to list
kruizeObjectList.addAll(mKruizeExperimentMap.values());
}
} else {
error = true;
sendErrorResponse(
response,
new Exception(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_CLUSTER_NAME_EXCPTN),
HttpServletResponse.SC_BAD_REQUEST,
String.format(AnalyzerErrorConstants.APIErrors.ListRecommendationsAPI.INVALID_CLUSTER_NAME_MSG, clusterName)
);
}
} else {
try {
new ExperimentDBService().loadAllExperimentsAndRecommendations(mKruizeExperimentMap);
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/autotune/analyzer/utils/AnalyzerErrorConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ public static final class ListRecommendationsAPI {
public static final String INVALID_EXPERIMENT_NAME_MSG = "Given experiment name - \" %s \" is not valid";
public static final String INVALID_QUERY_PARAM = "The query param(s) - \" %s \" is/are invalid";
public static final String INVALID_QUERY_PARAM_VALUE = "The query param value(s) is/are invalid";
public static final String INVALID_CLUSTER_NAME_EXCPTN = "Invalid Cluster Name";
public static final String INVALID_CLUSTER_NAME_MSG = "Given cluster name - \" %s \" is not valid";

private ListRecommendationsAPI() {

Expand Down
7 changes: 5 additions & 2 deletions src/main/java/com/autotune/database/dao/ExperimentDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,16 @@ public interface ExperimentDAO {
// Load a single experiment based on experimentName
List<KruizeExperimentEntry> loadExperimentByName(String experimentName) throws Exception;

// Load all results for a particular experimentName
//Load all experiments of a particular clusterName
List<KruizeExperimentEntry> loadExperimentsByClusterName(String clusterName) throws Exception;

// Load all results for a particular experimentName
List<KruizeResultsEntry> loadResultsByExperimentName(String experimentName, String cluster_name, Timestamp interval_start_time, Timestamp interval_end_time) throws Exception;

// Load all recommendations of a particular experiment
List<KruizeRecommendationEntry> loadRecommendationsByExperimentName(String experimentName) throws Exception;

// Load all recommendations of a particular cluster
List<KruizeRecommendationEntry> loadRecommendationsByClusterName(String clusterName) throws Exception;

// Load a single Performance Profile based on name
List<KruizePerformanceProfileEntry> loadPerformanceProfileByName(String performanceProfileName) throws Exception;
Expand Down
42 changes: 42 additions & 0 deletions src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,28 @@ public List<KruizeExperimentEntry> loadExperimentByName(String experimentName) t
}
return entries;
}
@Override
public List<KruizeExperimentEntry> loadExperimentsByClusterName(String clusterName) throws Exception {
//todo load only experimentStatus=inprogress , playback may not require completed experiments
List<KruizeExperimentEntry> entries = null;
String statusValue = "failure";
Timer.Sample timerLoadExpName = Timer.start(MetricsConfig.meterRegistry());
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_EXPERIMENTS_BY_CLUSTER_NAME, KruizeExperimentEntry.class)
.setParameter("clusterName", clusterName).list();
statusValue = "success";
} catch (Exception e) {
LOGGER.error("Not able to load experiments of cluster {} due to {}", clusterName, e.getMessage());
throw new Exception("Error while loading existing experiments of specified cluster from database due to : " + e.getMessage());
} finally {
if (null != timerLoadExpName) {
MetricsConfig.timerLoadExpName = MetricsConfig.timerBLoadExpName.tag("status", statusValue).register(MetricsConfig.meterRegistry());
timerLoadExpName.stop(MetricsConfig.timerLoadExpName);
}

}
return entries;
}

@Override
public List<KruizeResultsEntry> loadResultsByExperimentName(String experimentName, String cluster_name, Timestamp calculated_start_time, Timestamp interval_end_time) throws Exception {
Expand Down Expand Up @@ -561,6 +583,26 @@ public List<KruizeRecommendationEntry> loadRecommendationsByExperimentName(Strin
return recommendationEntries;
}

public List<KruizeRecommendationEntry> loadRecommendationsByClusterName(String clusterName) throws Exception {
List<KruizeRecommendationEntry> recommendationEntries = null;
String statusValue = "failure";
Timer.Sample timerLoadRecExpName = Timer.start(MetricsConfig.meterRegistry());
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
recommendationEntries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_RECOMMENDATIONS_BY_CLUSTER_NAME, KruizeRecommendationEntry.class)
.setParameter("clusterName", clusterName).list();
statusValue = "success";
} catch (Exception e) {
LOGGER.error("Not able to load recommendations due to {}", e.getMessage());
throw new Exception("Error while loading existing recommendations from database due to : " + e.getMessage());
} finally {
if (null != timerLoadRecExpName) {
MetricsConfig.timerLoadRecExpName = MetricsConfig.timerBLoadRecExpName.tag("status", statusValue).register(MetricsConfig.meterRegistry());
timerLoadRecExpName.stop(MetricsConfig.timerLoadRecExpName);
}
}
return recommendationEntries;
}

@Override
public KruizeRecommendationEntry loadRecommendationsByExperimentNameAndDate(String experimentName, String cluster_name, Timestamp interval_end_time) throws Exception {
KruizeRecommendationEntry recommendationEntries = null;
Expand Down
18 changes: 9 additions & 9 deletions src/main/java/com/autotune/database/helper/DBConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ public class DBConstants {
public static final class SQLQUERY {
public static final String SELECT_FROM_EXPERIMENTS = "from KruizeExperimentEntry";
public static final String SELECT_FROM_EXPERIMENTS_BY_EXP_NAME = "from KruizeExperimentEntry k WHERE k.experiment_name = :experimentName";
public static final String SELECT_FROM_EXPERIMENTS_BY_CLUSTER_NAME = "from KruizeExperimentEntry k WHERE k.cluster_name = :clusterName";
public static final String SELECT_FROM_RESULTS = "from KruizeResultsEntry";
public static final String SELECT_FROM_RESULTS_BY_EXP_NAME = "from KruizeResultsEntry k WHERE k.experiment_name = :experimentName";
public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_DATE_RANGE_AND_LIMIT =
Expand All @@ -18,25 +19,24 @@ public static final class SQLQUERY {
KruizeConstants.JSONKeys.CALCULATED_START_TIME,
KruizeConstants.JSONKeys.INTERVAL_END_TIME);
public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_START_END_TIME = String.format("from KruizeResultsEntry k " +
"WHERE k.experiment_name = :%s and k.interval_start_time >= :%s and " +
"WHERE k.cluster_name = :%s k.experiment_name = :%s and k.interval_start_time >= :%s and " +
"k.interval_end_time <= :%s ",
KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.INTERVAL_START_TIME, KruizeConstants.JSONKeys.INTERVAL_END_TIME);
KruizeConstants.JSONKeys.CLUSTER_NAME, KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.INTERVAL_START_TIME, KruizeConstants.JSONKeys.INTERVAL_END_TIME);
public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_END_TIME = String.format(
"from KruizeResultsEntry k WHERE " +
"k.cluster_name = :%s and " +
"k.experiment_name = :%s " +
"and k.interval_end_time = :%s ",
KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.INTERVAL_END_TIME);
KruizeConstants.JSONKeys.CLUSTER_NAME, KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.INTERVAL_END_TIME);
public static final String SELECT_FROM_RESULTS_BY_EXP_NAME_AND_MAX_END_TIME = String.format(
"from KruizeResultsEntry k WHERE " +
"k.cluster_name = :%s and " +
"k.experiment_name = :%s and " +
"k.interval_end_time = (SELECT MAX(e.interval_end_time) FROM KruizeResultsEntry e where e.experiment_name = :%s ) ",
KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.EXPERIMENT_NAME);
KruizeConstants.JSONKeys.CLUSTER_NAME, KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.EXPERIMENT_NAME);
public static final String SELECT_FROM_RECOMMENDATIONS_BY_EXP_NAME = String.format("from KruizeRecommendationEntry k WHERE k.experiment_name = :experimentName");
public static final String SELECT_FROM_RECOMMENDATIONS_BY_EXP_NAME_AND_END_TIME = String.format(
"from KruizeRecommendationEntry k WHERE " +
"k.experiment_name = :%s and " +
"k.interval_end_time= :%s ",
KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.INTERVAL_END_TIME);
public static final String SELECT_FROM_RECOMMENDATIONS_BY_EXP_NAME_AND_END_TIME = String.format("from KruizeRecommendationEntry k WHERE k.cluster_name= :%s and k.experiment_name = :%s and k.interval_end_time= :%s", KruizeConstants.JSONKeys.CLUSTER_NAME, KruizeConstants.JSONKeys.EXPERIMENT_NAME, KruizeConstants.JSONKeys.INTERVAL_END_TIME);
public static final String SELECT_FROM_RECOMMENDATIONS_BY_CLUSTER_NAME = "from KruizeRecommendationEntry k WHERE k.cluster_name = :clusterName";
public static final String SELECT_FROM_RECOMMENDATIONS = "from KruizeRecommendationEntry";
public static final String SELECT_FROM_PERFORMANCE_PROFILE = "from KruizePerformanceProfileEntry";
public static final String SELECT_FROM_PERFORMANCE_PROFILE_BY_NAME = "from KruizePerformanceProfileEntry k WHERE k.name = :name";
Expand Down
55 changes: 55 additions & 0 deletions src/main/java/com/autotune/database/service/ExperimentDBService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,28 @@ public void loadRecommendationsFromDBByName(Map<String, KruizeObject> mainKruize
}
}

public void loadRecommendationsFromDBByClusterName(Map<String, KruizeObject> mainKruizeExperimentMap, String clusterName) throws Exception {
ExperimentInterface experimentInterface = new ExperimentInterfaceImpl();
// Load Recommendations from DB and save to local
List<KruizeRecommendationEntry> recommendationEntries = experimentDAO.loadRecommendationsByClusterName(clusterName);
if (null != recommendationEntries && !recommendationEntries.isEmpty()) {
List<ListRecommendationsAPIObject> recommendationsAPIObjects
= null;
try {
recommendationsAPIObjects = DBHelpers.Converters.KruizeObjectConverters
.convertRecommendationEntryToRecommendationAPIObject(recommendationEntries);
} catch (InvalidConversionOfRecommendationEntryException e) {
e.printStackTrace();
}
if (null != recommendationsAPIObjects && !recommendationsAPIObjects.isEmpty()) {
experimentInterface.addRecommendationsToLocalStorage(mainKruizeExperimentMap,
recommendationsAPIObjects,
true);
}
}
}


public ValidationOutputData addExperimentToDB(CreateExperimentAPIObject createExperimentAPIObject) {
ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null);
try {
Expand Down Expand Up @@ -303,6 +325,34 @@ public void loadExperimentFromDBByName(Map<String, KruizeObject> mainKruizeExper
}
}
}
public void loadExperimentsFromDBByClusterName(Map<String, KruizeObject> mainKruizeExperimentMap, String clusterName) throws Exception {
ExperimentInterface experimentInterface = new ExperimentInterfaceImpl();
List<KruizeExperimentEntry> entries = experimentDAO.loadExperimentsByClusterName(clusterName);
if (null != entries && !entries.isEmpty()) {
List<CreateExperimentAPIObject> createExperimentAPIObjects = DBHelpers.Converters.KruizeObjectConverters.convertExperimentEntryToCreateExperimentAPIObject(entries);
if (null != createExperimentAPIObjects && !createExperimentAPIObjects.isEmpty()) {
List<KruizeObject> kruizeExpList = new ArrayList<>();

int failureThreshHold = createExperimentAPIObjects.size();
int failureCount = 0;
for (CreateExperimentAPIObject createExperimentAPIObject : createExperimentAPIObjects) {
KruizeObject kruizeObject = Converters.KruizeObjectConverters.convertCreateExperimentAPIObjToKruizeObject(createExperimentAPIObject);
if (null != kruizeObject) {
kruizeExpList.add(kruizeObject);
} else {
failureCount++;
}
}
if (failureThreshHold > 0 && failureCount == failureThreshHold) {
throw new Exception("Failed to load experiments for cluster " + clusterName + " from the database. No valid experiments found.");
}
experimentInterface.addExperimentToLocalStorage(mainKruizeExperimentMap, kruizeExpList);
}
}
}





public void loadExperimentAndResultsFromDBByName(Map<String, KruizeObject> mainKruizeExperimentMap, String experimentName) throws Exception {
Expand All @@ -319,6 +369,11 @@ public void loadExperimentAndRecommendationsFromDBByName(Map<String, KruizeObjec
loadRecommendationsFromDBByName(mainKruizeExperimentMap, experimentName);
}

public void loadExperimentsAndRecommendationsFromDBByClusterName(Map<String, KruizeObject> mainKruizeExperimentMap, String clusterName) throws Exception {
loadExperimentsFromDBByClusterName(mainKruizeExperimentMap, clusterName);
loadRecommendationsFromDBByClusterName(mainKruizeExperimentMap, clusterName);
}

public void loadPerformanceProfileFromDBByName(Map<String, PerformanceProfile> performanceProfileMap, String performanceProfileName) throws Exception {
List<KruizePerformanceProfileEntry> entries = experimentDAO.loadPerformanceProfileByName(performanceProfileName);
if (null != entries && !entries.isEmpty()) {
Expand Down
Loading