From 783838d323e04dceda112db1c1506f3e64e14af9 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 4 Sep 2024 19:19:37 +0530 Subject: [PATCH 1/5] Add list recomm json schema for local monitoring --- .../list_reco_json_local_monitoring_schema.py | 446 ++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 tests/scripts/helpers/list_reco_json_local_monitoring_schema.py diff --git a/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py b/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py new file mode 100644 index 000000000..d29d654ec --- /dev/null +++ b/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py @@ -0,0 +1,446 @@ +list_reco_json_local_monitoring_schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "cluster_name": { "type": "string" }, + "kubernetes_objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "name": { "type": "string" }, + "namespace": { "type": "string" }, + "containers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "container_image_name": { "type": "string" }, + "container_name": { "type": "string" }, + "recommendations": { + "type": "object", + "properties": { + "version": { "type": "string" }, + "notifications": { + "type": "object", + "items": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "message": { "type": "string" }, + "code": { "type": "number" } + }, + "required": ["type", "message", "code"] + } + }, + "data": { + "type": "object", + "patternProperties": { + "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z$": { + "type": "object", + "properties": { + "notifications": { + "type": "object", + "items": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "message": { "type": "string" }, + "code": { "type": "number" } + }, + "required": ["type", "message", "code"] + } + }, + "monitoring_end_time": { "type": "string" }, + "current": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": [] + }, + "limits": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": ["memory", "cpu"] + } + }, + "required": [] + }, + "recommendation_terms": { + "type": "object", + "properties": { + "short_term": { + "type": "object", + "properties": { + "notifications": { + "type": "object", + "items": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "message": { "type": "string" }, + "code": { "type": "number" } + }, + "required": ["type", "message", "code"] + } + }, + "monitoring_start_time": { "type": "string" }, + "duration_in_hours": { "type": "number" }, + "recommendation_engines": { + "type": "object", + "properties": { + "cost": { + "type": "object", + "properties": { + "pods_count": { "type": "number" }, + "confidence_level": { "type": "number" }, + "config": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": [] + }, + "limits": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": [] + } + }, + "required": ["requests", "limits"] + }, + "variation": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": [] + }, + "limits": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": [] + } + }, + "required": ["requests", "limits"] + }, + "notifications": { + "type": "object", + "items": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "message": { "type": "string" }, + "code": { "type": "number" } + }, + "required": ["type", "message", "code"] + } + } + }, + "required": ["pods_count", "confidence_level", "config", "variation", "notifications"] + }, + "performance": { + "type": "object", + "properties": { + "monitoring_start_time": { "type": "string" }, + "pods_count": { "type": "number" }, + "confidence_level": { "type": "number" }, + "config": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": ["memory", "cpu"] + }, + "limits": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": ["memory", "cpu"] + } + }, + "required": ["requests", "limits"] + }, + "variation": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": ["memory", "cpu"] + }, + "limits": { + "type": "object", + "properties": { + "memory": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + }, + "cpu": { + "type": "object", + "properties": { + "amount": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["amount", "format"] + } + }, + "required": ["memory", "cpu"] + } + }, + "required": ["requests", "limits"] + }, + "notifications": { + "type": "object", + "items": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "message": { "type": "string" }, + "code": { "type": "number" } + }, + "required": ["type", "message", "code"] + } + } + }, + "required": [] + } + }, + "required": [] + }, + "plots": { + "type": "object", + "properties": { + "datapoints": { "type": "number" }, + "plots_data": { + "type": "object", + "patternProperties": { + "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z$": { + "type": "object", + "properties": { + "cpuUsage": { + "type": "object", + "properties": { + "min": { "type": "number" }, + "q1": { "type": "number" }, + "median": { "type": "number" }, + "q3": { "type": "number" }, + "max": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["min", "q1", "median", "q3", "max", "format"] + }, + "memoryUsage": { + "type": "object", + "properties": { + "min": { "type": "number" }, + "q1": { "type": "number" }, + "median": { "type": "number" }, + "q3": { "type": "number" }, + "max": { "type": "number" }, + "format": { "type": "string" } + }, + "required": ["min", "q1", "median", "q3", "max", "format"] + }, + }, + "required": [] + } + }, + "required": [] + } + }, + "required": ["datapoints", "plots_data"] + } + }, + "required": [] + } + }, + "required": [] + } + }, + "required": [] + } + }, + "required": [] + } + }, + "required": ["version", "notifications", "data"] + } + }, + "required": ["container_image_name", "container_name", "recommendations"] + } + } + }, + "required": ["type", "name", "namespace", "containers"] + } + }, + "version": { "type": "string" }, + "experiment_name": { "type": "string" } + }, + "required": ["cluster_name", "kubernetes_objects", "version", "experiment_name"] + } +} + From de2c393eb69a933b4cab860a97e2cafa91c81c25 Mon Sep 17 00:00:00 2001 From: Shreya Date: Wed, 4 Sep 2024 19:21:16 +0530 Subject: [PATCH 2/5] Add functions to validate recommendation values for local monitoring --- tests/scripts/helpers/utils.py | 139 +++++++++++++++++- .../test_local_monitoring_e2e_workflow.py | 13 +- 2 files changed, 146 insertions(+), 6 deletions(-) diff --git a/tests/scripts/helpers/utils.py b/tests/scripts/helpers/utils.py index f66505ff7..482ce7ea5 100644 --- a/tests/scripts/helpers/utils.py +++ b/tests/scripts/helpers/utils.py @@ -446,6 +446,17 @@ def validate_reco_json(create_exp_json, update_results_json, list_reco_json, exp validate_kubernetes_obj(create_exp_kubernetes_obj, update_results_kubernetes_obj, update_results_json, list_reco_kubernetes_obj, expected_duration_in_hours, test_name) +def validate_local_monitoring_reco_json(create_exp_json, list_reco_json, expected_duration_in_hours=None, test_name=None): + # Validate experiment + assert create_exp_json["version"] == list_reco_json["version"] + assert create_exp_json["experiment_name"] == list_reco_json["experiment_name"] + assert create_exp_json["cluster_name"] == list_reco_json["cluster_name"] + + # Validate kubernetes objects + create_exp_kubernetes_obj = create_exp_json["kubernetes_objects"][0] + list_reco_kubernetes_obj = list_reco_json["kubernetes_objects"][0] + validate_local_monitoring_kubernetes_obj(create_exp_kubernetes_obj, list_reco_kubernetes_obj, expected_duration_in_hours, + test_name) def validate_list_exp_results_count(expected_results_count, list_exp_json): @@ -506,6 +517,27 @@ def validate_kubernetes_obj(create_exp_kubernetes_obj, update_results_kubernetes validate_container(update_results_container, update_results_json, list_reco_container, expected_duration_in_hours, test_name) +def validate_local_monitoring_kubernetes_obj(create_exp_kubernetes_obj, + list_reco_kubernetes_obj, expected_duration_in_hours, test_name): + # Validate type, name, namespace + assert list_reco_kubernetes_obj["type"] == create_exp_kubernetes_obj["type"] + assert list_reco_kubernetes_obj["name"] == create_exp_kubernetes_obj["name"] + assert list_reco_kubernetes_obj["namespace"] == create_exp_kubernetes_obj["namespace"] + + exp_containers_length = len(create_exp_kubernetes_obj["containers"]) + list_reco_containers_length = len(list_reco_kubernetes_obj["containers"]) + + + # Validate if all the containers are present + for i in range(exp_containers_length): + list_reco_container = None + + for j in range(list_reco_containers_length): + if list_reco_kubernetes_obj["containers"][j]["container_name"] == \ + create_exp_kubernetes_obj["containers"][i]["container_name"]: + list_reco_container = list_reco_kubernetes_obj["containers"][j] + create_exp_container = create_exp_kubernetes_obj["containers"][i] + validate_local_monitoring_container(create_exp_container, list_reco_container, expected_duration_in_hours, test_name) def validate_container(update_results_container, update_results_json, list_reco_container, expected_duration_in_hours, test_name): @@ -610,6 +642,91 @@ def validate_container(update_results_container, update_results_json, list_reco_ result = check_if_recommendations_are_present(list_reco_container["recommendations"]) assert result == False, f"Recommendations notifications does not contain the expected message - {NOT_ENOUGH_DATA_MSG}" +def validate_local_monitoring_container(create_exp_container, list_reco_container, expected_duration_in_hours, test_name): + # Validate container image name and container name + if create_exp_container != None and list_reco_container != None: + assert list_reco_container["container_image_name"] == create_exp_container["container_image_name"], \ + f"Container image names did not match! Actual - {list_reco_container['container_image_name']} Expected - {create_exp_container['container_image_name']}" + + assert list_reco_container["container_name"] == create_exp_container["container_name"], \ + f"Container names did not match! Acutal = {list_reco_container['container_name']} Expected - {create_exp_container['container_name']}" + + + if expected_duration_in_hours == None: + duration_in_hours = 0.0 + else: + duration_in_hours = expected_duration_in_hours + + interval_end_time = list(list_reco_container['recommendations']['data'].keys())[0] + interval_start_time = list_reco_container['recommendations']['data'][interval_end_time]['recommendation_terms']['short_term']['monitoring_start_time'] + print(f"interval_end_time = {interval_end_time} interval_start_time = {interval_start_time}") + + if check_if_recommendations_are_present(list_reco_container["recommendations"]): + terms_obj = list_reco_container["recommendations"]["data"][interval_end_time]["recommendation_terms"] + current_config = list_reco_container["recommendations"]["data"][interval_end_time]["current"] + + duration_terms = {'short_term': 4, 'medium_term': 7, 'long_term': 15} + for term in duration_terms.keys(): + if check_if_recommendations_are_present(terms_obj[term]): + print(f"reco present for term {term}") + # Validate timestamps [deprecated as monitoring end time is moved to higher level] + # assert cost_obj[term]["monitoring_end_time"] == interval_end_time, \ + # f"monitoring end time {cost_obj[term]['monitoring_end_time']} did not match end timestamp {interval_end_time}" + + # Validate the precision of the valid duration + duration = terms_obj[term]["duration_in_hours"] + assert validate_duration_in_hours_decimal_precision(duration), f"The value '{duration}' for " \ + f"'{term}' has more than two decimal places" + + monitoring_start_time = term_based_start_time(interval_end_time, term) + assert terms_obj[term]["monitoring_start_time"] == monitoring_start_time, \ + f"actual = {terms_obj[term]['monitoring_start_time']} expected = {monitoring_start_time}" + + # Validate duration in hrs + if expected_duration_in_hours is None: + duration_in_hours = set_duration_based_on_terms(duration_in_hours, term, + interval_start_time, interval_end_time) + + if test_name is not None: + + if MEDIUM_TERM_TEST in test_name and term == MEDIUM_TERM: + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + elif SHORT_TERM_TEST in test_name and term == SHORT_TERM: + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + elif LONG_TERM_TEST in test_name and term == LONG_TERM: + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + else: + print( + f"Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}") + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + duration_in_hours = set_duration_based_on_terms(duration_in_hours, term, interval_start_time, + interval_end_time) + + # Get engine objects + engines_list = ["cost", "performance"] + + # Extract recommendation engine objects + recommendation_engines_object = None + if "recommendation_engines" in terms_obj[term]: + recommendation_engines_object = terms_obj[term]["recommendation_engines"] + if recommendation_engines_object is not None: + for engine_entry in engines_list: + if engine_entry in terms_obj[term]["recommendation_engines"]: + engine_obj = terms_obj[term]["recommendation_engines"][engine_entry] + validate_config_local_monitoring(engine_obj["config"]) + validate_variation(current_config, engine_obj["config"], engine_obj["variation"]) + # validate Plots data + validate_plots(terms_obj, duration_terms, term) + # verify that plots isn't generated in case of no recommendations + else: + assert PLOTS not in terms_obj[term], f"Expected plots to be absent in case of no recommendations" + else: + data = list_reco_container["recommendations"]["data"] + assert len(data) == 0, f"Data is not empty! Length of data - Actual = {len(data)} expected = 0" def validate_plots(terms_obj, duration_terms, term): plots = terms_obj[term][PLOTS] @@ -663,6 +780,20 @@ def validate_config(reco_config, metrics): assert reco_config[usage]["memory"][ "format"] == memory_format_type, f"memory format in recommendation config is {reco_config[usage]['memory']['format']} instead of {memory_format_type}" +def validate_config_local_monitoring(reco_config): + cpu_format_type = "cores" + memory_format_type = "bytes" + + usage_list = ["requests", "limits"] + for usage in usage_list: + assert reco_config[usage]["cpu"][ + "amount"] > 0, f"cpu amount in recommendation config is {reco_config[usage]['cpu']['amount']}" + assert reco_config[usage]["cpu"][ + "format"] == cpu_format_type, f"cpu format in recommendation config is {reco_config[usage]['cpu']['format']} instead of {cpu_format_type}" + assert reco_config[usage]["memory"][ + "amount"] > 0, f"cpu amount in recommendation config is {reco_config[usage]['memory']['amount']}" + assert reco_config[usage]["memory"][ + "format"] == memory_format_type, f"memory format in recommendation config is {reco_config[usage]['memory']['format']} instead of {memory_format_type}" def check_if_recommendations_are_present(cost_obj): notifications = cost_obj["notifications"] @@ -830,13 +961,13 @@ def validate_variation(current_config: dict, recommended_config: dict, variation current_cpu_value = 0 current_memory_value = 0 if CPU_KEY in recommended_requests: - if CPU_KEY in current_requests and AMOUNT_KEY in current_requests[CPU_KEY]: + if current_requests is not None and CPU_KEY in current_requests and AMOUNT_KEY in current_requests[CPU_KEY]: current_cpu_value = current_requests[CPU_KEY][AMOUNT_KEY] assert variation_requests[CPU_KEY][AMOUNT_KEY] == recommended_requests[CPU_KEY][ AMOUNT_KEY] - current_cpu_value assert variation_requests[CPU_KEY][FORMAT_KEY] == recommended_requests[CPU_KEY][FORMAT_KEY] if MEMORY_KEY in recommended_requests: - if MEMORY_KEY in current_requests and AMOUNT_KEY in current_requests[MEMORY_KEY]: + if current_requests is not None and MEMORY_KEY in current_requests and AMOUNT_KEY in current_requests[MEMORY_KEY]: current_memory_value = current_requests[MEMORY_KEY][AMOUNT_KEY] assert variation_requests[MEMORY_KEY][AMOUNT_KEY] == recommended_requests[MEMORY_KEY][ AMOUNT_KEY] - current_memory_value @@ -845,12 +976,12 @@ def validate_variation(current_config: dict, recommended_config: dict, variation current_cpu_value = 0 current_memory_value = 0 if CPU_KEY in recommended_limits: - if CPU_KEY in current_limits and AMOUNT_KEY in current_limits[CPU_KEY]: + if current_limits is not None and CPU_KEY in current_limits and AMOUNT_KEY in current_limits[CPU_KEY]: current_cpu_value = current_limits[CPU_KEY][AMOUNT_KEY] assert variation_limits[CPU_KEY][AMOUNT_KEY] == recommended_limits[CPU_KEY][AMOUNT_KEY] - current_cpu_value assert variation_limits[CPU_KEY][FORMAT_KEY] == recommended_limits[CPU_KEY][FORMAT_KEY] if MEMORY_KEY in recommended_limits: - if MEMORY_KEY in current_limits and AMOUNT_KEY in current_limits[MEMORY_KEY]: + if current_limits is not None and MEMORY_KEY in current_limits and AMOUNT_KEY in current_limits[MEMORY_KEY]: current_memory_value = current_limits[MEMORY_KEY][AMOUNT_KEY] assert variation_limits[MEMORY_KEY][AMOUNT_KEY] == recommended_limits[MEMORY_KEY][ AMOUNT_KEY] - current_memory_value diff --git a/tests/scripts/local_monitoring_tests/rest_apis/test_local_monitoring_e2e_workflow.py b/tests/scripts/local_monitoring_tests/rest_apis/test_local_monitoring_e2e_workflow.py index edb1203c1..049978187 100644 --- a/tests/scripts/local_monitoring_tests/rest_apis/test_local_monitoring_e2e_workflow.py +++ b/tests/scripts/local_monitoring_tests/rest_apis/test_local_monitoring_e2e_workflow.py @@ -33,6 +33,7 @@ from helpers.utils import clone_repo from helpers.utils import apply_tfb_load from helpers.utils import wait_for_container_to_complete +from helpers.utils import validate_local_monitoring_reco_json from helpers.list_metadata_json_validate import * from helpers.list_metadata_json_schema import * from helpers.list_metadata_json_verbose_true_schema import * @@ -40,6 +41,7 @@ from helpers.list_metric_profiles_validate import * from helpers.list_metric_profiles_without_parameters_schema import * from helpers.short_term_list_reco_json_schema import * +from helpers.list_reco_json_local_monitoring_schema import * from helpers.list_reco_json_validate import * from helpers.import_metadata_json_validate import * @@ -193,9 +195,12 @@ def test_list_recommendations_multiple_exps_for_datasource_workloads(cluster_typ list_reco_json = response.json() # Validate the json against the json schema - errorMsg = validate_list_reco_json(list_reco_json, list_reco_json_schema) + errorMsg = validate_list_reco_json(list_reco_json, list_reco_json_local_monitoring_schema) assert errorMsg == "" + # Validate the json values + tfb_exp_json = read_json_data_from_file(tfb_exp_json_file) + validate_local_monitoring_reco_json(tfb_exp_json[0], list_reco_json[0]) response = generate_recommendations(tfb_db_exp_name) assert response.status_code == SUCCESS_STATUS_CODE @@ -206,9 +211,13 @@ def test_list_recommendations_multiple_exps_for_datasource_workloads(cluster_typ list_reco_json = response.json() # Validate the json against the json schema - errorMsg = validate_list_reco_json(list_reco_json, list_reco_json_schema) + errorMsg = validate_list_reco_json(list_reco_json, list_reco_json_local_monitoring_schema) assert errorMsg == "" + # Validate the json values + tfb_db_exp_json = read_json_data_from_file(tfb_db_exp_json_file) + validate_local_monitoring_reco_json(tfb_db_exp_json[0], list_reco_json[0]) + # Delete tfb experiment response = delete_experiment(tfb_exp_json_file) print("delete exp = ", response.status_code) From a7ccbf5456f31e3d52bce474a44d78f895eeb5f2 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 5 Sep 2024 11:08:25 +0530 Subject: [PATCH 3/5] Remove required cpu and memory fields for performance reco values --- .../helpers/list_reco_json_local_monitoring_schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py b/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py index d29d654ec..23187207c 100644 --- a/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py +++ b/tests/scripts/helpers/list_reco_json_local_monitoring_schema.py @@ -276,7 +276,7 @@ "required": ["amount", "format"] } }, - "required": ["memory", "cpu"] + "required": [] }, "limits": { "type": "object", @@ -298,7 +298,7 @@ "required": ["amount", "format"] } }, - "required": ["memory", "cpu"] + "required": [] } }, "required": ["requests", "limits"] @@ -326,7 +326,7 @@ "required": ["amount", "format"] } }, - "required": ["memory", "cpu"] + "required": [] }, "limits": { "type": "object", @@ -348,7 +348,7 @@ "required": ["amount", "format"] } }, - "required": ["memory", "cpu"] + "required": [] } }, "required": ["requests", "limits"] From d1913cec405983e9da168e5a90e76460a4516e41 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 5 Sep 2024 14:08:55 +0530 Subject: [PATCH 4/5] Refactor container validation and check for notfication message --- tests/scripts/helpers/utils.py | 154 +++++++++++++++++---------------- 1 file changed, 80 insertions(+), 74 deletions(-) diff --git a/tests/scripts/helpers/utils.py b/tests/scripts/helpers/utils.py index 482ce7ea5..06c4995b5 100644 --- a/tests/scripts/helpers/utils.py +++ b/tests/scripts/helpers/utils.py @@ -657,76 +657,80 @@ def validate_local_monitoring_container(create_exp_container, list_reco_containe else: duration_in_hours = expected_duration_in_hours + if check_if_recommendations_are_present(list_reco_container["recommendations"]): interval_end_time = list(list_reco_container['recommendations']['data'].keys())[0] - interval_start_time = list_reco_container['recommendations']['data'][interval_end_time]['recommendation_terms']['short_term']['monitoring_start_time'] - print(f"interval_end_time = {interval_end_time} interval_start_time = {interval_start_time}") - - if check_if_recommendations_are_present(list_reco_container["recommendations"]): - terms_obj = list_reco_container["recommendations"]["data"][interval_end_time]["recommendation_terms"] - current_config = list_reco_container["recommendations"]["data"][interval_end_time]["current"] - - duration_terms = {'short_term': 4, 'medium_term': 7, 'long_term': 15} - for term in duration_terms.keys(): - if check_if_recommendations_are_present(terms_obj[term]): - print(f"reco present for term {term}") - # Validate timestamps [deprecated as monitoring end time is moved to higher level] - # assert cost_obj[term]["monitoring_end_time"] == interval_end_time, \ - # f"monitoring end time {cost_obj[term]['monitoring_end_time']} did not match end timestamp {interval_end_time}" - - # Validate the precision of the valid duration - duration = terms_obj[term]["duration_in_hours"] - assert validate_duration_in_hours_decimal_precision(duration), f"The value '{duration}' for " \ - f"'{term}' has more than two decimal places" - - monitoring_start_time = term_based_start_time(interval_end_time, term) - assert terms_obj[term]["monitoring_start_time"] == monitoring_start_time, \ - f"actual = {terms_obj[term]['monitoring_start_time']} expected = {monitoring_start_time}" - - # Validate duration in hrs - if expected_duration_in_hours is None: - duration_in_hours = set_duration_based_on_terms(duration_in_hours, term, - interval_start_time, interval_end_time) - - if test_name is not None: - - if MEDIUM_TERM_TEST in test_name and term == MEDIUM_TERM: - assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ - f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" - elif SHORT_TERM_TEST in test_name and term == SHORT_TERM: - assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ - f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" - elif LONG_TERM_TEST in test_name and term == LONG_TERM: - assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ - f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" - else: - print( - f"Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}") + print(f"interval_end_time = {interval_end_time}") + + terms_obj = list_reco_container["recommendations"]["data"][interval_end_time]["recommendation_terms"] + current_config = list_reco_container["recommendations"]["data"][interval_end_time]["current"] + + duration_terms = {'short_term': 4, 'medium_term': 7, 'long_term': 15} + for term in duration_terms.keys(): + if check_if_recommendations_are_present(terms_obj[term]): + print(f"reco present for term {term}") + # Validate timestamps [deprecated as monitoring end time is moved to higher level] + # assert cost_obj[term]["monitoring_end_time"] == interval_end_time, \ + # f"monitoring end time {cost_obj[term]['monitoring_end_time']} did not match end timestamp {interval_end_time}" + + interval_start_time = list_reco_container['recommendations']['data'][interval_end_time]['recommendation_terms'][term]['monitoring_start_time'] + # Validate the precision of the valid duration + duration = terms_obj[term]["duration_in_hours"] + assert validate_duration_in_hours_decimal_precision(duration), f"The value '{duration}' for " \ + f"'{term}' has more than two decimal places" + + monitoring_start_time = term_based_start_time(interval_end_time, term) + assert terms_obj[term]["monitoring_start_time"] == monitoring_start_time, \ + f"actual = {terms_obj[term]['monitoring_start_time']} expected = {monitoring_start_time}" + + # Validate duration in hrs + if expected_duration_in_hours is None: + duration_in_hours = set_duration_based_on_terms(duration_in_hours, term, + interval_start_time, interval_end_time) + + if test_name is not None: + + if MEDIUM_TERM_TEST in test_name and term == MEDIUM_TERM: + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + elif SHORT_TERM_TEST in test_name and term == SHORT_TERM: + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + elif LONG_TERM_TEST in test_name and term == LONG_TERM: assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" - duration_in_hours = set_duration_based_on_terms(duration_in_hours, term, interval_start_time, - interval_end_time) - - # Get engine objects - engines_list = ["cost", "performance"] - - # Extract recommendation engine objects - recommendation_engines_object = None - if "recommendation_engines" in terms_obj[term]: - recommendation_engines_object = terms_obj[term]["recommendation_engines"] - if recommendation_engines_object is not None: - for engine_entry in engines_list: - if engine_entry in terms_obj[term]["recommendation_engines"]: - engine_obj = terms_obj[term]["recommendation_engines"][engine_entry] - validate_config_local_monitoring(engine_obj["config"]) - validate_variation(current_config, engine_obj["config"], engine_obj["variation"]) - # validate Plots data - validate_plots(terms_obj, duration_terms, term) - # verify that plots isn't generated in case of no recommendations else: - assert PLOTS not in terms_obj[term], f"Expected plots to be absent in case of no recommendations" - else: - data = list_reco_container["recommendations"]["data"] - assert len(data) == 0, f"Data is not empty! Length of data - Actual = {len(data)} expected = 0" + print( + f"Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}") + assert terms_obj[term]["duration_in_hours"] == duration_in_hours, \ + f"Duration in hours did not match! Actual = {terms_obj[term]['duration_in_hours']} expected = {duration_in_hours}" + duration_in_hours = set_duration_based_on_terms(duration_in_hours, term, interval_start_time, + interval_end_time) + + # Get engine objects + engines_list = ["cost", "performance"] + + # Extract recommendation engine objects + recommendation_engines_object = None + if "recommendation_engines" in terms_obj[term]: + recommendation_engines_object = terms_obj[term]["recommendation_engines"] + if recommendation_engines_object is not None: + for engine_entry in engines_list: + if engine_entry in terms_obj[term]["recommendation_engines"]: + engine_obj = terms_obj[term]["recommendation_engines"][engine_entry] + validate_config_local_monitoring(engine_obj["config"]) + validate_variation(current_config, engine_obj["config"], engine_obj["variation"]) + # validate Plots data + validate_plots(terms_obj, duration_terms, term) + # verify that plots isn't generated in case of no recommendations + else: + assert PLOTS not in terms_obj[term], f"Expected plots to be absent in case of no recommendations" + else: + notifications = list_reco_container["recommendations"]["notifications"] + if NOTIFICATION_CODE_FOR_NOT_ENOUGH_DATA in notifications: + assert notifications[NOTIFICATION_CODE_FOR_NOT_ENOUGH_DATA]["message"] == NOT_ENOUGH_DATA_MSG + + data = list_reco_container["recommendations"]["data"] + assert len(data) == 0, f"Data is not empty! Length of data - Actual = {len(data)} expected = 0" def validate_plots(terms_obj, duration_terms, term): plots = terms_obj[term][PLOTS] @@ -786,14 +790,16 @@ def validate_config_local_monitoring(reco_config): usage_list = ["requests", "limits"] for usage in usage_list: - assert reco_config[usage]["cpu"][ - "amount"] > 0, f"cpu amount in recommendation config is {reco_config[usage]['cpu']['amount']}" - assert reco_config[usage]["cpu"][ - "format"] == cpu_format_type, f"cpu format in recommendation config is {reco_config[usage]['cpu']['format']} instead of {cpu_format_type}" - assert reco_config[usage]["memory"][ - "amount"] > 0, f"cpu amount in recommendation config is {reco_config[usage]['memory']['amount']}" - assert reco_config[usage]["memory"][ - "format"] == memory_format_type, f"memory format in recommendation config is {reco_config[usage]['memory']['format']} instead of {memory_format_type}" + if "cpu" in reco_config[usage]: + assert reco_config[usage]["cpu"][ + "amount"] > 0, f"cpu amount in recommendation config is {reco_config[usage]['cpu']['amount']}" + assert reco_config[usage]["cpu"][ + "format"] == cpu_format_type, f"cpu format in recommendation config is {reco_config[usage]['cpu']['format']} instead of {cpu_format_type}" + if "memory" in reco_config[usage]: + assert reco_config[usage]["memory"][ + "amount"] > 0, f"cpu amount in recommendation config is {reco_config[usage]['memory']['amount']}" + assert reco_config[usage]["memory"][ + "format"] == memory_format_type, f"memory format in recommendation config is {reco_config[usage]['memory']['format']} instead of {memory_format_type}" def check_if_recommendations_are_present(cost_obj): notifications = cost_obj["notifications"] From a50b40354f3c4978bd3c7b12eda7ae74254a1558 Mon Sep 17 00:00:00 2001 From: Shreya Date: Thu, 5 Sep 2024 14:14:54 +0530 Subject: [PATCH 5/5] Validate variation of reco values and check for cpu idle notification --- tests/scripts/helpers/utils.py | 82 +++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/tests/scripts/helpers/utils.py b/tests/scripts/helpers/utils.py index 06c4995b5..dbc84ac75 100644 --- a/tests/scripts/helpers/utils.py +++ b/tests/scripts/helpers/utils.py @@ -718,7 +718,7 @@ def validate_local_monitoring_container(create_exp_container, list_reco_containe if engine_entry in terms_obj[term]["recommendation_engines"]: engine_obj = terms_obj[term]["recommendation_engines"][engine_entry] validate_config_local_monitoring(engine_obj["config"]) - validate_variation(current_config, engine_obj["config"], engine_obj["variation"]) + validate_variation_local_monitoring(current_config, engine_obj["config"], engine_obj["variation"], engine_obj) # validate Plots data validate_plots(terms_obj, duration_terms, term) # verify that plots isn't generated in case of no recommendations @@ -948,6 +948,78 @@ def validate_variation(current_config: dict, recommended_config: dict, variation recommended_limits: dict = None variation_limits: dict = None + if REQUESTS_KEY in current_config: + current_requests = current_config[REQUESTS_KEY] + if LIMITS_KEY in current_config: + current_limits = current_config[LIMITS_KEY] + + if REQUESTS_KEY in recommended_config: + recommended_requests = recommended_config[REQUESTS_KEY] + if LIMITS_KEY in recommended_config: + recommended_limits = recommended_config[LIMITS_KEY] + + if REQUESTS_KEY in variation_config: + variation_requests = variation_config[REQUESTS_KEY] + if LIMITS_KEY in variation_config: + variation_limits = variation_config[LIMITS_KEY] + + if recommended_requests is not None: + current_cpu_value = 0 + current_memory_value = 0 + if CPU_KEY in recommended_requests: + if CPU_KEY in current_requests and AMOUNT_KEY in current_requests[CPU_KEY]: + current_cpu_value = current_requests[CPU_KEY][AMOUNT_KEY] + assert variation_requests[CPU_KEY][AMOUNT_KEY] == recommended_requests[CPU_KEY][ + AMOUNT_KEY] - current_cpu_value + assert variation_requests[CPU_KEY][FORMAT_KEY] == recommended_requests[CPU_KEY][FORMAT_KEY] + if MEMORY_KEY in recommended_requests: + if MEMORY_KEY in current_requests and AMOUNT_KEY in current_requests[MEMORY_KEY]: + current_memory_value = current_requests[MEMORY_KEY][AMOUNT_KEY] + assert variation_requests[MEMORY_KEY][AMOUNT_KEY] == recommended_requests[MEMORY_KEY][ + AMOUNT_KEY] - current_memory_value + assert variation_requests[MEMORY_KEY][FORMAT_KEY] == recommended_requests[MEMORY_KEY][FORMAT_KEY] + if recommended_limits is not None: + current_cpu_value = 0 + current_memory_value = 0 + if CPU_KEY in recommended_limits: + if CPU_KEY in current_limits and AMOUNT_KEY in current_limits[CPU_KEY]: + current_cpu_value = current_limits[CPU_KEY][AMOUNT_KEY] + assert variation_limits[CPU_KEY][AMOUNT_KEY] == recommended_limits[CPU_KEY][AMOUNT_KEY] - current_cpu_value + assert variation_limits[CPU_KEY][FORMAT_KEY] == recommended_limits[CPU_KEY][FORMAT_KEY] + if MEMORY_KEY in recommended_limits: + if MEMORY_KEY in current_limits and AMOUNT_KEY in current_limits[MEMORY_KEY]: + current_memory_value = current_limits[MEMORY_KEY][AMOUNT_KEY] + assert variation_limits[MEMORY_KEY][AMOUNT_KEY] == recommended_limits[MEMORY_KEY][ + AMOUNT_KEY] - current_memory_value + assert variation_limits[MEMORY_KEY][FORMAT_KEY] == recommended_limits[MEMORY_KEY][FORMAT_KEY] + + +def validate_variation_local_monitoring(current_config: dict, recommended_config: dict, variation_config: dict, engine_obj): + # Check structure + assert check_if_dict_has_same_keys(recommended_config, variation_config) == True + + # Create temporary dict if it's none jus to make process easier + if current_config == None: + current_config = {} + + # Check values + REQUESTS_KEY = "requests" + LIMITS_KEY = "limits" + CPU_KEY = "cpu" + MEMORY_KEY = "memory" + AMOUNT_KEY = "amount" + FORMAT_KEY = "format" + + # Initialise requests holders + current_requests: dict = None + recommended_requests: dict = None + variation_requests: dict = None + + # Initialise limits holders + current_limits: dict = None + recommended_limits: dict = None + variation_limits: dict = None + if REQUESTS_KEY in current_config: current_requests = current_config[REQUESTS_KEY] if LIMITS_KEY in current_config: @@ -972,6 +1044,10 @@ def validate_variation(current_config: dict, recommended_config: dict, variation assert variation_requests[CPU_KEY][AMOUNT_KEY] == recommended_requests[CPU_KEY][ AMOUNT_KEY] - current_cpu_value assert variation_requests[CPU_KEY][FORMAT_KEY] == recommended_requests[CPU_KEY][FORMAT_KEY] + else: + assert NOTIFICATION_CODE_FOR_CPU_RECORDS_ARE_IDLE in engine_obj["notifications"] + assert engine_obj["notifications"][NOTIFICATION_CODE_FOR_CPU_RECORDS_ARE_IDLE]["message"] == NOTIFICATION_CODE_FOR_CPU_RECORDS_ARE_IDLE_MESSAGE + if MEMORY_KEY in recommended_requests: if current_requests is not None and MEMORY_KEY in current_requests and AMOUNT_KEY in current_requests[MEMORY_KEY]: current_memory_value = current_requests[MEMORY_KEY][AMOUNT_KEY] @@ -986,6 +1062,10 @@ def validate_variation(current_config: dict, recommended_config: dict, variation current_cpu_value = current_limits[CPU_KEY][AMOUNT_KEY] assert variation_limits[CPU_KEY][AMOUNT_KEY] == recommended_limits[CPU_KEY][AMOUNT_KEY] - current_cpu_value assert variation_limits[CPU_KEY][FORMAT_KEY] == recommended_limits[CPU_KEY][FORMAT_KEY] + else: + assert NOTIFICATION_CODE_FOR_CPU_RECORDS_ARE_IDLE in engine_obj["notifications"] + assert engine_obj["notifications"][NOTIFICATION_CODE_FOR_CPU_RECORDS_ARE_IDLE]["message"] == NOTIFICATION_CODE_FOR_CPU_RECORDS_ARE_IDLE_MESSAGE + if MEMORY_KEY in recommended_limits: if current_limits is not None and MEMORY_KEY in current_limits and AMOUNT_KEY in current_limits[MEMORY_KEY]: current_memory_value = current_limits[MEMORY_KEY][AMOUNT_KEY]