From 0173279c271fe5ce412d4d23181a297f66b64f34 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 22 Dec 2023 11:40:13 +0530 Subject: [PATCH] add validation of recommendation json for multiple clusters --- .../remote_monitoring_tests/helpers/utils.py | 8 + .../rest_apis/test_list_recommendations.py | 188 +++++++++--------- 2 files changed, 104 insertions(+), 92 deletions(-) diff --git a/tests/scripts/remote_monitoring_tests/helpers/utils.py b/tests/scripts/remote_monitoring_tests/helpers/utils.py index aad97c4a4..d687b501c 100644 --- a/tests/scripts/remote_monitoring_tests/helpers/utils.py +++ b/tests/scripts/remote_monitoring_tests/helpers/utils.py @@ -675,3 +675,11 @@ def validate_variation(current_config: dict, recommended_config: dict, variation assert variation_limits[MEMORY_KEY][AMOUNT_KEY] == recommended_limits[MEMORY_KEY][ AMOUNT_KEY] - current_memory_value assert variation_limits[MEMORY_KEY][FORMAT_KEY] == recommended_limits[MEMORY_KEY][FORMAT_KEY] + +def validate_reco_json_multiple_clusters(exp_jsons_dir, update_results_json, list_reco_json, expected_duration_in_hours=None, + test_name=None, num_cluster=0): + # Validate experiment + for i in range(len(list_reco_json)): + create_exp_json_file = exp_jsons_dir + "/create_exp_" + str(num_cluster) + "_" + str(i) + ".json" + create_exp_json = read_json_data_from_file(create_exp_json_file) + validate_reco_json(create_exp_json[0], update_results_json[i], list_reco_json[i], expected_duration_in_hours) \ No newline at end of file diff --git a/tests/scripts/remote_monitoring_tests/rest_apis/test_list_recommendations.py b/tests/scripts/remote_monitoring_tests/rest_apis/test_list_recommendations.py index 5e133cc9e..1762254bf 100644 --- a/tests/scripts/remote_monitoring_tests/rest_apis/test_list_recommendations.py +++ b/tests/scripts/remote_monitoring_tests/rest_apis/test_list_recommendations.py @@ -1758,83 +1758,84 @@ def test_list_recommendations_single_cluster_multiple_exps(cluster_type): # Form the Kruize service URL form_kruize_url(cluster_type) - for i in range(num_clusters): + update_results_json = [] + for i in range(num_exps): - for j in range(num_exps): + # Create experiment using the specified json + create_exp_json_file = exp_jsons_dir + "/create_exp_" + str(0) + "_" + str(i) + ".json" - # Create experiment using the specified json - create_exp_json_file = exp_jsons_dir + "/create_exp_" + str(i) + "_" + str(j) + ".json" + response = delete_experiment(create_exp_json_file) + print("delete exp = ", response.status_code) - response = delete_experiment(create_exp_json_file) - print("delete exp = ", response.status_code) + response = create_experiment(create_exp_json_file) - response = create_experiment(create_exp_json_file) + data = response.json() + json_data = json.load(open(create_exp_json_file)) + obj_type = json_data[0]['kubernetes_objects'][0]["type"] - data = response.json() - json_data = json.load(open(create_exp_json_file)) - obj_type = json_data[0]['kubernetes_objects'][0]["type"] + print(f"kubernetes_object_type = {obj_type}") - print(f"kubernetes_object_type = {obj_type}") + print("message = ", data['message']) - print("message = ", data['message']) + assert response.status_code == SUCCESS_STATUS_CODE + assert data['status'] == SUCCESS_STATUS + assert data['message'] == CREATE_EXP_SUCCESS_MSG - assert response.status_code == SUCCESS_STATUS_CODE - assert data['status'] == SUCCESS_STATUS - assert data['message'] == CREATE_EXP_SUCCESS_MSG + result_json_arr = [] + start_time = None + end_time = None - result_json_arr = [] - start_time = None - end_time = None + # Update results for the experiment + result_json_file = result_jsons_dir + "/result_" + str(0) + "_" + str(i) + ".json" + result_json = read_json_data_from_file(result_json_file) + if start_time is None: + start_time = result_json[num_res-1]['interval_start_time'] + end_time = result_json[num_res-1]['interval_end_time'] - # Update results for the experiment - result_json_file = result_jsons_dir + "/result_" + str(i) + "_" + str(j) + ".json" - result_json = read_json_data_from_file(result_json_file) - if start_time is None: - start_time = result_json[num_res-1]['interval_start_time'] - end_time = result_json[num_res-1]['interval_end_time'] + response = update_results(result_json_file) + data = response.json() - response = update_results(result_json_file) - data = response.json() + print("message = ", data['message']) + assert response.status_code == SUCCESS_STATUS_CODE + assert data['status'] == SUCCESS_STATUS + assert data[ + 'message'] == UPDATE_RESULTS_SUCCESS_MSG, f"expected message = {UPDATE_RESULTS_SUCCESS_MSG} actual message = {data['message']}" - print("message = ", data['message']) - assert response.status_code == SUCCESS_STATUS_CODE - assert data['status'] == SUCCESS_STATUS - assert data[ - 'message'] == UPDATE_RESULTS_SUCCESS_MSG, f"expected message = {UPDATE_RESULTS_SUCCESS_MSG} actual message = {data['message']}" + result_json_data = read_json_data_from_file(result_json_file) + result_json_arr.append(result_json_data[num_res-1]) - result_json_data = read_json_data_from_file(result_json_file) - result_json_arr.append(result_json_data[num_res-1]) + # Get the cluster name and experiment name + json_data = json.load(open(create_exp_json_file)) + cluster_name = json_data[0]['cluster_name'] + experiment_name = json_data[0]['experiment_name'] - # Get the cluster name and experiment name - json_data = json.load(open(create_exp_json_file)) - cluster_name = json_data[0]['cluster_name'] - experiment_name = json_data[0]['experiment_name'] + # Update Recommendations + response = update_recommendations(experiment_name, None, end_time) + data = response.json() + assert response.status_code == SUCCESS_STATUS_CODE + assert data[0]['experiment_name'] == experiment_name + assert data[0]['kubernetes_objects'][0]['containers'][0]['recommendations']['notifications'][NOTIFICATION_CODE_FOR_RECOMMENDATIONS_AVAILABLE][ + 'message'] == RECOMMENDATIONS_AVAILABLE - # Update Recommendations - response = update_recommendations(experiment_name, None, end_time) - data = response.json() - assert response.status_code == SUCCESS_STATUS_CODE - assert data[0]['experiment_name'] == experiment_name - assert data[0]['kubernetes_objects'][0]['containers'][0]['recommendations']['notifications'][NOTIFICATION_CODE_FOR_RECOMMENDATIONS_AVAILABLE][ - 'message'] == RECOMMENDATIONS_AVAILABLE + update_results_json.append(result_json_arr) - # Invoke list recommendations for the specified experiment - response = list_recommendations(cluster_name=cluster_name) - assert response.status_code == SUCCESS_200_STATUS_CODE + cluster_name = "cluster-one-division-bell_0" + # Invoke list recommendations for the specified experiment + response = list_recommendations(cluster_name=cluster_name) + assert response.status_code == SUCCESS_200_STATUS_CODE - list_reco_json = response.json() - print("list_reco_json = ", list_reco_json) - # Validate the json against the json schema - errorMsg = validate_list_reco_json(list_reco_json, list_reco_json_schema) - assert errorMsg == "" + list_reco_json = response.json() - create_exp_json = read_json_data_from_file(create_exp_json_file) - update_results_json = [] - update_results_json.append(result_json_arr[len(result_json_arr) - 1]) + # Sort the JSON data by 'experiment_name' + sorted_list_reco_json = sorted(list_reco_json, key=lambda x: x['experiment_name']) - # Expected duration in hours is 24h as for short term only 24h plus or minus 30s of data is considered to generate recommendations - expected_duration_in_hours = SHORT_TERM_DURATION_IN_HRS_MAX - validate_reco_json(create_exp_json[0], update_results_json, list_reco_json[0], expected_duration_in_hours) + # Validate the json against the json schema + errorMsg = validate_list_reco_json(sorted_list_reco_json, list_reco_json_schema) + assert errorMsg == "" + + # Expected duration in hours is 24h as for short term only 24h plus or minus 30s of data is considered to generate recommendations + expected_duration_in_hours = SHORT_TERM_DURATION_IN_HRS_MAX + validate_reco_json_multiple_clusters(exp_jsons_dir, update_results_json, sorted_list_reco_json, expected_duration_in_hours) # Delete the experiments for i in range(num_clusters): @@ -1851,7 +1852,7 @@ def test_list_recommendations_multiple_clusters_multiple_exps(cluster_type): """ num_clusters = 6 num_exps = 6 - num_res = 120 + num_res = 100 split = False split_count = 1 @@ -1870,7 +1871,7 @@ def test_list_recommendations_multiple_clusters_multiple_exps(cluster_type): form_kruize_url(cluster_type) for i in range(num_clusters): - + update_results_json = [] for j in range(num_exps): # Create experiment using the specified json @@ -1896,25 +1897,25 @@ def test_list_recommendations_multiple_clusters_multiple_exps(cluster_type): result_json_arr = [] start_time = None end_time = None - for k in range(num_res): - # Update results for the experiment - result_json_file = result_jsons_dir + "/result_" + str(i) + "_" + str(j) + "_" + str(k) + ".json" - result_json = read_json_data_from_file(result_json_file) - if start_time is None: - start_time = result_json[0]['interval_start_time'] - end_time = result_json[0]['interval_end_time'] - - response = update_results(result_json_file) - data = response.json() - print("message = ", data['message']) - assert response.status_code == SUCCESS_STATUS_CODE - assert data['status'] == SUCCESS_STATUS - assert data[ - 'message'] == UPDATE_RESULTS_SUCCESS_MSG, f"expected message = {UPDATE_RESULTS_SUCCESS_MSG} actual message = {data['message']}" + # Update results for the experiment + result_json_file = result_jsons_dir + "/result_" + str(i) + "_" + str(j) + ".json" + result_json = read_json_data_from_file(result_json_file) + if start_time is None: + start_time = result_json[num_res-1]['interval_start_time'] + end_time = result_json[num_res-1]['interval_end_time'] + + response = update_results(result_json_file) + data = response.json() + + print("message = ", data['message']) + assert response.status_code == SUCCESS_STATUS_CODE + assert data['status'] == SUCCESS_STATUS + assert data[ + 'message'] == UPDATE_RESULTS_SUCCESS_MSG, f"expected message = {UPDATE_RESULTS_SUCCESS_MSG} actual message = {data['message']}" - result_json_data = read_json_data_from_file(result_json_file) - result_json_arr.append(result_json_data[0]) + result_json_data = read_json_data_from_file(result_json_file) + result_json_arr.append(result_json_data[num_res-1]) # Get the cluster name and experiment name json_data = json.load(open(create_exp_json_file)) @@ -1926,26 +1927,29 @@ def test_list_recommendations_multiple_clusters_multiple_exps(cluster_type): data = response.json() assert response.status_code == SUCCESS_STATUS_CODE assert data[0]['experiment_name'] == experiment_name - assert data[0]['kubernetes_objects'][0]['containers'][0]['recommendations']['notifications']['112101'][ - 'message'] == 'Cost Recommendations Available' + assert data[0]['cluster_name'] == cluster_name + assert data[0]['kubernetes_objects'][0]['containers'][0]['recommendations']['notifications'][NOTIFICATION_CODE_FOR_RECOMMENDATIONS_AVAILABLE][ + 'message'] == RECOMMENDATIONS_AVAILABLE - # Invoke list recommendations for the specified experiment - response = list_recommendations(experiment_name) - assert response.status_code == SUCCESS_200_STATUS_CODE + update_results_json.append(result_json_arr) - list_reco_json = response.json() + cluster_name = "cluster-one-division-bell" + "_" + str(i) + # Invoke list recommendations for the specified cluster + response = list_recommendations(cluster_name=cluster_name) + assert response.status_code == SUCCESS_200_STATUS_CODE - # Validate the json against the json schema - errorMsg = validate_list_reco_json(list_reco_json, list_reco_json_schema) - assert errorMsg == "" + list_reco_json = response.json() - create_exp_json = read_json_data_from_file(create_exp_json_file) - update_results_json = [] - update_results_json.append(result_json_arr[len(result_json_arr) - 1]) + # Sort the JSON data by 'experiment_name' + sorted_list_reco_json = sorted(list_reco_json, key=lambda x: x['experiment_name']) - # Expected duration in hours is 24h as for short term only 24h plus or minus 30s of data is considered to generate recommendations - expected_duration_in_hours = SHORT_TERM_DURATION_IN_HRS_MAX - validate_reco_json(create_exp_json[0], update_results_json, list_reco_json[0], expected_duration_in_hours) + # Validate the json against the json schema + errorMsg = validate_list_reco_json(sorted_list_reco_json, list_reco_json_schema) + assert errorMsg == "" + + # Expected duration in hours is 24h as for short term only 24h plus or minus 30s of data is considered to generate recommendations + expected_duration_in_hours = SHORT_TERM_DURATION_IN_HRS_MAX + validate_reco_json_multiple_clusters(exp_jsons_dir, update_results_json, sorted_list_reco_json, expected_duration_in_hours, test_name=None, num_cluster=i) # Delete the experiments for i in range(num_clusters):