Fix percentile (#36)

* split percentile into its bucket and value separately

* remove unused params references

* add reporting of sysbench version to debug output

* linting

* Automatic upate of README.md by arcaflow-docsgen arcabot

* improve schema metadata

* Automatic upate of README.md by arcaflow-docsgen arcabot

* string cleanups

---------

Co-authored-by: arcabot <[email protected]>

README.md

@@ -189,17 +189,20 @@ Run CPU performance test using the sysbench workload
         </details></td></tr>
 </tbody></table>
         </details><details><summary>LatencyAggregates (<code>object</code>)</summary>
-            <table><tbody><tr><th>Type:</th><td><code>object</code></td><tr><th>Properties</th><td><details><summary>P95thpercentile (<code>float</code>)</summary>
-        <table><tbody><tr><th>Name:</th><td>95th Percentile</td></tr><tr><th>Description:</th><td>95th percentile latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
-</tbody></table>
-        </details><details><summary>avg (<code>float</code>)</summary>
+            <table><tbody><tr><th>Type:</th><td><code>object</code></td><tr><th>Properties</th><td><details><summary>avg (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Average</td></tr><tr><th>Description:</th><td>Average Latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>max (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Maximum</td></tr><tr><th>Description:</th><td>Maximum Latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>min (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Minimum</td></tr><tr><th>Description:</th><td>Minimum latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
+</tbody></table>
+        </details><details><summary>percentile (<code>int</code>)</summary>
+        <table><tbody><tr><th>Name:</th><td>Percentile</td></tr><tr><th>Description:</th><td>Latency percentile selected for reporting</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>int</code></td>
+</tbody></table>
+        </details><details><summary>percentile_value (<code>float</code>)</summary>
+        <table><tbody><tr><th>Name:</th><td>Latency Percentile Value</td></tr><tr><th>Description:</th><td>Latency percentile value</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>sum (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Sum</td></tr><tr><th>Description:</th><td>Sum of latencies</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
@@ -543,17 +546,20 @@ Run the I/O test using the sysbench workload
         </details></td></tr>
 </tbody></table>
         </details><details><summary>LatencyAggregates (<code>object</code>)</summary>
-            <table><tbody><tr><th>Type:</th><td><code>object</code></td><tr><th>Properties</th><td><details><summary>P95thpercentile (<code>float</code>)</summary>
-        <table><tbody><tr><th>Name:</th><td>95th Percentile</td></tr><tr><th>Description:</th><td>95th percentile latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
-</tbody></table>
-        </details><details><summary>avg (<code>float</code>)</summary>
+            <table><tbody><tr><th>Type:</th><td><code>object</code></td><tr><th>Properties</th><td><details><summary>avg (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Average</td></tr><tr><th>Description:</th><td>Average Latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>max (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Maximum</td></tr><tr><th>Description:</th><td>Maximum Latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>min (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Minimum</td></tr><tr><th>Description:</th><td>Minimum latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
+</tbody></table>
+        </details><details><summary>percentile (<code>int</code>)</summary>
+        <table><tbody><tr><th>Name:</th><td>Percentile</td></tr><tr><th>Description:</th><td>Latency percentile selected for reporting</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>int</code></td>
+</tbody></table>
+        </details><details><summary>percentile_value (<code>float</code>)</summary>
+        <table><tbody><tr><th>Name:</th><td>Latency Percentile Value</td></tr><tr><th>Description:</th><td>Latency percentile value</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>sum (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Sum</td></tr><tr><th>Description:</th><td>Sum of latencies</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
@@ -841,17 +847,20 @@ Run the Memory functions speed test using the sysbench workload
                 <table><tbody><tr><th>Name:</th><td>Sysbench Memory Result Parameters</td></tr><tr><th>Description:</th><td>Result parameters for a successful sysbench Memory workload execution</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>reference[SysbenchMemoryResultParams]</code></td><tr><th>Referenced object:</th><td>SysbenchMemoryResultParams</td></tr></tbody></table>
             </details></td></tr>
 <tr><td colspan="2"><details><summary><strong>Objects</strong></summary><details><summary>LatencyAggregates (<code>object</code>)</summary>
-            <table><tbody><tr><th>Type:</th><td><code>object</code></td><tr><th>Properties</th><td><details><summary>P95thpercentile (<code>float</code>)</summary>
-        <table><tbody><tr><th>Name:</th><td>95th Percentile</td></tr><tr><th>Description:</th><td>95th percentile latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
-</tbody></table>
-        </details><details><summary>avg (<code>float</code>)</summary>
+            <table><tbody><tr><th>Type:</th><td><code>object</code></td><tr><th>Properties</th><td><details><summary>avg (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Average</td></tr><tr><th>Description:</th><td>Average Latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>max (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Maximum</td></tr><tr><th>Description:</th><td>Maximum Latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>min (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Minimum</td></tr><tr><th>Description:</th><td>Minimum latency</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
+</tbody></table>
+        </details><details><summary>percentile (<code>int</code>)</summary>
+        <table><tbody><tr><th>Name:</th><td>Percentile</td></tr><tr><th>Description:</th><td>Latency percentile selected for reporting</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>int</code></td>
+</tbody></table>
+        </details><details><summary>percentile_value (<code>float</code>)</summary>
+        <table><tbody><tr><th>Name:</th><td>Latency Percentile Value</td></tr><tr><th>Description:</th><td>Latency percentile value</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>
 </tbody></table>
         </details><details><summary>sum (<code>float</code>)</summary>
         <table><tbody><tr><th>Name:</th><td>Sum</td></tr><tr><th>Description:</th><td>Sum of latencies</td></tr><tr><th>Required:</th><td>Yes</td></tr><tr><th>Type:</th><td><code>float</code></td>

arcaflow_plugin_sysbench/sysbench_plugin.py

@@ -35,7 +35,10 @@ def parse_output(output):
             key, value = line.split(":")
 
             if key[0].isdigit():
-                key = "P" + key
+                percentile_value = value
+                value = re.match(r"([0-9]+)", key).group(0)
+                key = "percentile"
+
             if value == "":
                 key = re.sub(r"\((.*?)\)", "", key)
                 if "options" in key or "General" in key:
@@ -57,7 +60,7 @@ def parse_output(output):
                     tops = tops.replace("persecond)", "")
                     dictionary["Totaloperations"] = float(to)
                     dictionary["Totaloperationspersecond"] = float(tops)
-                elif value.isnumeric():
+                elif "percentile" not in key and value.isnumeric():
                     dictionary[key] = float(value)
                 else:
                     dictionary[key] = value
@@ -71,7 +74,10 @@ def parse_output(output):
                     dictionary[section][key] = {}
                     dictionary[section][key]["avg"] = float(avg)
                     dictionary[section][key]["stddev"] = float(stddev)
-                elif value.isnumeric():
+                elif "percentile" in key:
+                    dictionary[section][key] = int(value)
+                    dictionary[section]["percentile_value"] = percentile_value
+                elif "percentile" not in key and value.isnumeric():
                     dictionary[section][key] = float(value)
                 else:
                     # replace / and , with _ for fileio test
@@ -90,7 +96,7 @@ def parse_output(output):
     return sysbench_output, sysbench_results
 
 
-def run_sysbench(params, flags, operation, test_mode="run"):
+def run_sysbench(flags, operation, test_mode="run"):
     try:
         cmd = ["sysbench"]
         cmd = cmd + flags + [operation, test_mode]
@@ -116,6 +122,24 @@ def run_sysbench(params, flags, operation, test_mode="run"):
         return output, results
 
 
+def get_sysbench_version():
+    try:
+        cmd = ["sysbench", "--version"]
+        version = (
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            .strip()
+            .decode("utf-8")
+        )
+    except subprocess.CalledProcessError as error:
+        raise Exception(
+            error.returncode,
+            "{} failed with return code {}:\n{}".format(
+                error.cmd[0], error.returncode, error.output
+            ),
+        ) from error
+    return version
+
+
 @plugin.step(
     id="sysbenchcpu",
     name="Sysbench CPU Workload",
@@ -125,6 +149,7 @@ def run_sysbench(params, flags, operation, test_mode="run"):
 def RunSysbenchCpu(
     params: SysbenchCpuInputParams,
 ) -> typing.Tuple[str, typing.Union[WorkloadResultsCpu, WorkloadError]]:
+    print(f"Sysbench version is: {get_sysbench_version()}")
     print("==>> Running sysbench CPU workload ...")
 
     serialized_params = sysbench_cpu_input_schema.serialize(params)
@@ -134,7 +159,7 @@ def RunSysbenchCpu(
         cpu_flags.append(f"--{param}={value}")
 
     try:
-        output, results = run_sysbench(params, cpu_flags, "cpu")
+        output, results = run_sysbench(cpu_flags, "cpu")
     except Exception as error:
         return "error", WorkloadError(error.args[0], error.args[1])
 
@@ -149,14 +174,13 @@ def RunSysbenchCpu(
 @plugin.step(
     id="sysbenchmemory",
     name="Sysbench Memory Workload",
-    description=(
-        "Run the Memory functions speed test using the sysbench workload"
-    ),
+    description=("Run the Memory functions speed test using the sysbench workload"),
     outputs={"success": WorkloadResultsMemory, "error": WorkloadError},
 )
 def RunSysbenchMemory(
     params: SysbenchMemoryInputParams,
 ) -> typing.Tuple[str, typing.Union[WorkloadResultsMemory, WorkloadError]]:
+    print(f"Sysbench version is: {get_sysbench_version()}")
     print("==>> Running sysbench Memory workload ...")
 
     serialized_params = sysbench_memory_input_schema.serialize(params)
@@ -166,7 +190,7 @@ def RunSysbenchMemory(
         memory_flags.append(f"--{param}={value}")
 
     try:
-        output, results = run_sysbench(params, memory_flags, "memory")
+        output, results = run_sysbench(memory_flags, "memory")
     except Exception as error:
         return "error", WorkloadError(error.args[0], error.args[1])
 
@@ -187,6 +211,7 @@ def RunSysbenchMemory(
 def RunSysbenchIo(
     params: SysbenchIoInputParams,
 ) -> typing.Tuple[str, typing.Union[WorkloadResultsIo, WorkloadError]]:
+    print(f"Sysbench version is: {get_sysbench_version()}")
     print("==>> Running sysbench I/O workload ...")
 
     serialized_params = sysbench_io_input_schema.serialize(params)
@@ -196,9 +221,9 @@ def RunSysbenchIo(
         io_flags.append(f"--{param}={value}")
 
     try:
-        run_sysbench(params, io_flags, "fileio", "prepare")
-        output, results = run_sysbench(params, io_flags, "fileio", "run")
-        run_sysbench(params, io_flags, "fileio", "cleanup")
+        run_sysbench(io_flags, "fileio", "prepare")
+        output, results = run_sysbench(io_flags, "fileio", "run")
+        run_sysbench(io_flags, "fileio", "cleanup")
     except Exception as error:
         return "error", WorkloadError(error.args[0], error.args[1])
 
@@ -213,8 +238,6 @@ def RunSysbenchIo(
 if __name__ == "__main__":
     sys.exit(
         plugin.run(
-            plugin.build_schema(
-                RunSysbenchCpu, RunSysbenchMemory, RunSysbenchIo
-            )
+            plugin.build_schema(RunSysbenchCpu, RunSysbenchMemory, RunSysbenchIo)
         )
     )

arcaflow_plugin_sysbench/sysbench_schema.py

@@ -198,9 +198,7 @@ class SysbenchCpuInputParams(CommonInputParameters):
         typing.Optional[int],
         schema.id("cpu-max-prime"),
         schema.name("CPU max prime"),
-        schema.description(
-            "The upper limit of the number of prime numbers generated"
-        ),
+        schema.description("The upper limit of the number of prime numbers generated"),
     ] = 10000
 
 
@@ -280,9 +278,7 @@ class SysbenchIoInputParams(CommonInputParameters):
         typing.Optional[FileTestMode],
         schema.id("file-test-mode"),
         schema.name("File Test Mode"),
-        schema.description(
-            "Test mode {seqwr, seqrewr, seqrd, rndrd, rndwr, rndrw}"
-        ),
+        schema.description("Test mode {seqwr, seqrewr, seqrd, rndrd, rndwr, rndrw}"),
     ] = FileTestMode.SEQWR
 
     file_io_mode: typing.Annotated[
@@ -296,9 +292,7 @@ class SysbenchIoInputParams(CommonInputParameters):
         typing.Optional[int],
         schema.id("file-async-backlog"),
         schema.name("File Async Backlog"),
-        schema.description(
-            "Number of asynchronous operatons to queue per thread"
-        ),
+        schema.description("Number of asynchronous operatons to queue per thread"),
     ] = 128
 
     file_extra_flags: typing.Annotated[
@@ -376,10 +370,15 @@ class LatencyAggregates:
         schema.name("Maximum"),
         schema.description("Maximum Latency"),
     ]
-    P95thpercentile: typing.Annotated[
+    percentile: typing.Annotated[
+        int,
+        schema.name("Percentile"),
+        schema.description("Latency percentile selected for reporting"),
+    ]
+    percentile_value: typing.Annotated[
         float,
-        schema.name("95th Percentile"),
-        schema.description("95th percentile latency"),
+        schema.name("Latency Percentile Value"),
+        schema.description("Latency percentile value"),
     ]
     sum: typing.Annotated[
         float,
@@ -518,8 +517,7 @@ class SysbenchMemoryOutput:
         float,
         schema.name("Total operations per second"),
         schema.description(
-            "Total number of operations performed by the memory workload"
-            " per second"
+            "Total number of operations performed by the memory workload" " per second"
         ),
     ]
     Totaloperations: typing.Annotated[
@@ -648,9 +646,7 @@ class SysbenchIoResultParams:
 
 
 @dataclass
-class SysbenchMemoryOutputParams(
-    SysbenchCommonOutputParams, SysbenchMemoryOutput
-):
+class SysbenchMemoryOutputParams(SysbenchCommonOutputParams, SysbenchMemoryOutput):
     """
     This is the data structure for all output
     parameters returned by sysbench memory benchmark.
@@ -696,16 +692,14 @@ class WorkloadResultsCpu:
         SysbenchCpuOutputParams,
         schema.name("Sysbench Cpu Output Parameters"),
         schema.description(
-            "Ouptut parameters for a successful sysbench cpu workload"
-            " execution"
+            "Ouptut parameters for a successful sysbench cpu workload execution"
         ),
     ]
     sysbench_results: typing.Annotated[
         SysbenchCpuResultParams,
         schema.name("Sysbench Cpu Result Parameters"),
         schema.description(
-            "Result parameters for a successful sysbench cpu workload"
-            " execution"
+            "Result parameters for a successful sysbench cpu workload execution"
         ),
     ]
 
@@ -721,16 +715,14 @@ class WorkloadResultsMemory:
         SysbenchMemoryOutputParams,
         schema.name("Sysbench Memory Output Parameters"),
         schema.description(
-            "Ouptut parameters for a successful sysbench memory workload"
-            " execution"
+            "Ouptut parameters for a successful sysbench memory workload execution"
         ),
     ]
     sysbench_results: typing.Annotated[
         SysbenchMemoryResultParams,
         schema.name("Sysbench Memory Result Parameters"),
         schema.description(
-            "Result parameters for a successful sysbench Memory workload"
-            " execution"
+            "Result parameters for a successful sysbench Memory workload execution"
         ),
     ]
 
@@ -746,17 +738,15 @@ class WorkloadResultsIo:
         SysbenchIoOutputParams,
         schema.name("Sysbench Io Output Parameters"),
         schema.description(
-            "Ouptut parameters for a successful sysbench io workload"
-            " execution"
+            "Ouptut parameters for a successful sysbench io workload execution"
         ),
     ]
 
     sysbench_results: typing.Annotated[
         SysbenchIoResultParams,
         schema.name("Sysbench io Result Parameters"),
         schema.description(
-            "Result parameters for a successful io Memory workload"
-            " execution"
+            "Result parameters for a successful io Memory workload execution"
         ),
     ]
 
@@ -770,9 +760,7 @@ class WorkloadError:
     exit_code: typing.Annotated[
         int,
         schema.name("Exit Code"),
-        schema.description(
-            "Exit code returned by the program in case of a failure"
-        ),
+        schema.description("Exit code returned by the program in case of a failure"),
     ]
     error: typing.Annotated[
         str,
@@ -782,21 +770,11 @@ class WorkloadError:
 
 
 sysbench_cpu_input_schema = plugin.build_object_schema(SysbenchCpuInputParams)
-sysbench_memory_input_schema = plugin.build_object_schema(
-    SysbenchMemoryInputParams
-)
+sysbench_memory_input_schema = plugin.build_object_schema(SysbenchMemoryInputParams)
 sysbench_io_input_schema = plugin.build_object_schema(SysbenchIoInputParams)
-sysbench_cpu_output_schema = plugin.build_object_schema(
-    SysbenchCpuOutputParams
-)
-sysbench_cpu_results_schema = plugin.build_object_schema(
-    SysbenchCpuResultParams
-)
-sysbench_memory_output_schema = plugin.build_object_schema(
-    SysbenchMemoryOutputParams
-)
-sysbench_memory_results_schema = plugin.build_object_schema(
-    SysbenchMemoryResultParams
-)
+sysbench_cpu_output_schema = plugin.build_object_schema(SysbenchCpuOutputParams)
+sysbench_cpu_results_schema = plugin.build_object_schema(SysbenchCpuResultParams)
+sysbench_memory_output_schema = plugin.build_object_schema(SysbenchMemoryOutputParams)
+sysbench_memory_results_schema = plugin.build_object_schema(SysbenchMemoryResultParams)
 sysbench_io_output_schema = plugin.build_object_schema(SysbenchIoOutputParams)
 sysbench_io_results_schema = plugin.build_object_schema(SysbenchIoResultParams)