From e9a1a361809ddcd20c1d4dc8625d299e9c00856b Mon Sep 17 00:00:00 2001 From: nanitebased Date: Mon, 7 Nov 2022 10:54:02 +0100 Subject: [PATCH 1/5] Make CSV dialect compatible with Python 3.11 --- Helper/influx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helper/influx.py b/Helper/influx.py index 7ae495a..1caaa2f 100644 --- a/Helper/influx.py +++ b/Helper/influx.py @@ -57,7 +57,7 @@ class baseDialect(Dialect): doublequote = False skipinitialspace = False lineterminator = '\r\n' - quotechar = '' + quotechar = '"' quoting = QUOTE_NONE From 59576f79ab0a4c6962143cb0fdd2553ab1f5014d Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 9 Nov 2022 12:29:12 +0100 Subject: [PATCH 2/5] Add KPIs definition to TestCases --- Facility/Loader/testcase_loader.py | 33 ++++++++++++++++++++++++++++++ Facility/facility.py | 6 ++++++ 2 files changed, 39 insertions(+) diff --git a/Facility/Loader/testcase_loader.py b/Facility/Loader/testcase_loader.py index 960cdeb..b65e16c 100644 --- a/Facility/Loader/testcase_loader.py +++ b/Facility/Loader/testcase_loader.py @@ -18,12 +18,14 @@ def __init__(self, data: Dict): # V2 only self.Name: (str | None) = data.pop('Name', None) self.Sequence: List[Dict] = data.pop('Sequence', []) + self.KPIs: Dict[str, List[str]] = data.pop('KPIs', {}) class TestCaseLoader(Loader): testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} dashboards: Dict[str, List[DashboardPanel]] = {} + kpis: Dict[str, List[Tuple[str, str]]] = {} parameters: Dict[str, Tuple[str, str]] = {} # For use only while processing data, not necessary afterwards @classmethod @@ -85,6 +87,29 @@ def validateParameters(cls, defs: TestCaseData) -> [(Level, str)]: f"Cannot guarantee consistency.")) return validation + @classmethod + def validateKPIs(cls, key: str, defs: TestCaseData) -> [(Level, str)]: + kpis = [] + validation = [] + + try: + for measurement in sorted(defs.KPIs.keys()): + kpiList = defs.KPIs[measurement] + if not isinstance(kpiList, List): + validation.append( + (Level.ERROR, f"KPIs for '{measurement}' ({key}) are not a list. Found '{kpiList}'")) + elif len(kpiList) == 0: + validation.append( + (Level.ERROR, f"'{measurement}' ({key}) defines an empty listf of KPIs")) + else: + for kpi in sorted(kpiList): + kpis.append((measurement, kpi)) + except Exception as e: + validation.append((Level.ERROR, f"Could not read KPIs dictionary for testcase '{key}': {e}")) + + cls.kpis[key] = kpis + return validation + @classmethod def ProcessData(cls, data: Dict) -> [(Level, str)]: version = str(data.pop('Version', 1)) @@ -140,6 +165,9 @@ def processV2Data(cls, data: Dict) -> [(Level, str)]: validation.extend( cls.createDashboard(defs.Name, defs)) + validation.extend( + cls.validateKPIs(defs.Name, defs)) + validation.extend( cls.validateParameters(defs)) @@ -150,6 +178,7 @@ def Clear(cls): cls.testCases = {} cls.extra = {} cls.dashboards = {} + cls.kpis = {} cls.parameters = {} @classmethod @@ -160,6 +189,10 @@ def GetCurrentTestCases(cls): def GetCurrentTestCaseExtras(cls): return cls.extra + @classmethod + def GetCurrentTestCaseKPIs(cls): + return cls.kpis + @classmethod def GetCurrentDashboards(cls): return cls.dashboards diff --git a/Facility/facility.py b/Facility/facility.py index 1860745..72df8a4 100644 --- a/Facility/facility.py +++ b/Facility/facility.py @@ -24,6 +24,7 @@ class Facility: testCases: Dict[str, List[ActionInformation]] = {} extra: Dict[str, Dict[str, object]] = {} dashboards: Dict[str, List[DashboardPanel]] = {} + kpis: Dict[str, List[Tuple[str, str]]] = {} resources: Dict[str, Resource] = {} scenarios: Dict[str, Dict] = {} @@ -63,6 +64,7 @@ def Reload(cls): cls.testCases = TestCaseLoader.GetCurrentTestCases() cls.extra = TestCaseLoader.GetCurrentTestCaseExtras() cls.dashboards = TestCaseLoader.GetCurrentDashboards() + cls.kpis = TestCaseLoader.GetCurrentTestCaseKPIs() cls.ues = UeLoader.GetCurrentUEs() cls.scenarios = ScenarioLoader.GetCurrentScenarios() @@ -95,6 +97,10 @@ def GetTestCaseDashboards(cls, id: str) -> List[DashboardPanel]: def GetTestCaseExtra(cls, id: str) -> Dict[str, object]: return cls.extra.get(id, {}) + @classmethod + def GetTestCaseKPIs(cls, id: str) -> List[Tuple[str, str]]: + return cls.kpis.get(id, []) + @classmethod def BusyResources(cls) -> List[Resource]: return [res for res in cls.resources.values() if res.Locked] From 571f31516a07d46b6ac1ad29978d86ac995cd9ed Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 9 Nov 2022 12:29:56 +0100 Subject: [PATCH 3/5] Implement /kpis endpoint --- Scheduler/execution/routes.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index 6a653c7..0cb7a15 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -4,6 +4,8 @@ from Scheduler.execution import bp from typing import Union, Optional from Settings import Config +from Data import ExperimentDescriptor +from Facility import Facility from os.path import join, isfile, abspath @@ -126,6 +128,21 @@ def descriptor(executionId: int): return f"Execution {executionId} not found", 404 +@bp.route('/kpis') +def kpis(executionId: int): + execution = executionOrTombstone(executionId) + + if execution is not None: + kpis = [] + descriptor = ExperimentDescriptor(execution.JsonDescriptor) + for testcase in sorted(descriptor.TestCases): + kpis.extend(Facility.GetTestCaseKPIs(testcase)) + + return jsonify({"KPIs": kpis}) + else: + return f"Execution {executionId} not found", 404 + + @bp.route('nextExecutionId') def nextExecutionId(): return jsonify({'NextId': Status.PeekNextId()}) From 7bcce1e327cdf6c325ae54f7437dcc7c92a7c646 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 9 Nov 2022 13:04:24 +0100 Subject: [PATCH 4/5] Avoid returning duplicated KPIs --- Scheduler/execution/routes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index 0cb7a15..82b4f27 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -133,12 +133,12 @@ def kpis(executionId: int): execution = executionOrTombstone(executionId) if execution is not None: - kpis = [] + kpis = set() descriptor = ExperimentDescriptor(execution.JsonDescriptor) - for testcase in sorted(descriptor.TestCases): - kpis.extend(Facility.GetTestCaseKPIs(testcase)) + for testcase in descriptor.TestCases: + kpis.update(Facility.GetTestCaseKPIs(testcase)) - return jsonify({"KPIs": kpis}) + return jsonify({"KPIs": sorted(kpis)}) else: return f"Execution {executionId} not found", 404 From 733a195f9eb0ff0bd93f868411ed7ba758accfd7 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Wed, 9 Nov 2022 13:11:44 +0100 Subject: [PATCH 5/5] Update documentation --- CHANGELOG.md | 5 +++++ docs/2-1_FACILITY_CONFIGURATION.md | 3 +++ docs/2-2_TESTCASE_PARAMETERS.md | 13 +++++++++++++ docs/A1_ENDPOINTS.md | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf857ac..58cad6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +**09/11/2022** [Version 3.6.1] + - Allow defining a set of KPIs per TestCase + - Implement `/execution//kpis` endpoint + - Avoid exception when sending CSVs to InfluxDb on Python 3.11 + **10/10/2022** [Version 3.6.0] - Implemented Child tasks, flow control: diff --git a/docs/2-1_FACILITY_CONFIGURATION.md b/docs/2-1_FACILITY_CONFIGURATION.md index 895eab7..cf64263 100644 --- a/docs/2-1_FACILITY_CONFIGURATION.md +++ b/docs/2-1_FACILITY_CONFIGURATION.md @@ -91,6 +91,9 @@ Distributed: False Dashboard: {} ```` +V2 TestCases can also contain an optional `KPIs` field, which can be used to define a sub-set of results that are +considered *of interest*. See [TestCase Parameters](/docs/2-2_TESTCASE_PARAMETERS.md) for more information. + ##### - V1 TestCase (`Version: 1` or missing) TestCases using this format follow the same approach as for UE files. The following is an example V1 TestCase: diff --git a/docs/2-2_TESTCASE_PARAMETERS.md b/docs/2-2_TESTCASE_PARAMETERS.md index 3de5b7a..37c84e3 100644 --- a/docs/2-2_TESTCASE_PARAMETERS.md +++ b/docs/2-2_TESTCASE_PARAMETERS.md @@ -8,6 +8,19 @@ be added to the yaml description. These keys are: value is set to an empty list ('[]') the test case is considered public and will appear on the list of Custom experiments for all users of the Portal. If the list contains one or more email addresses, the test case will be visible only to the users with matching emails. + - `KPIs`: Optional dictionary that can be used for defining a sub-set of results, from all of the generated by the +TestCase, that can be considered *of interest*. The following is an example of the format: + ```yaml +"KPIs": + ping: # Each key refers to a 'measurement' (table) + - Success # Each key contains a list of 'kpi' (table columns) + - Delay + iPerf: + - Throughput + ``` +These values (as (`measurement`, `kpi`) pairs) can be retrieved, per experiment execution, by using the +`execution//kpis` endpoint. When multiple TestCases are included as part of an experiment the union +of all KPIs (duplicates removed) are returned. - `Parameters`: Dictionary of dictionaries, where each entry is defined as follows: ```yaml "": diff --git a/docs/A1_ENDPOINTS.md b/docs/A1_ENDPOINTS.md index 043dd12..3563e63 100644 --- a/docs/A1_ENDPOINTS.md +++ b/docs/A1_ENDPOINTS.md @@ -46,6 +46,14 @@ Returns a compressed file that includes the logs and all files generated by the Returns a copy of the Experiment Descriptor that was used to define the execution. +### [GET] '/execution//kpis' + +Returns a dictionary with a single `KPIs` key, containing a list of pairs (`measurement`, `kpi`) that are considered of +interest. + +> These values can be used as part of queries to the [Analytics Module](https://github.com/5genesis/Analytics), in order +> to extract a sub-set of important KPIs from all the generated measurements. + ### [DELETE] `/execution/` > *[GET] `/execution//cancel` (Deprecated)*