Merge pull request #81 from george0st/changes

Tune garbage collector usage & Generate outputs with 'only_new'

dev-requirements.txt

@@ -9,7 +9,7 @@ numpy>=1.26.4,<=2.1.2
 psutil>=5.9.0,<=6.0.0
 packaging>=21.0,<=24.1
 
-qgate_graph==1.4.26
+qgate_graph==1.4.27
 
 coverage>=7
 coverage-badge>=1

qgate_perf/parallel_executor.py

@@ -361,7 +361,7 @@ def run_bulk_executor(self,
                 performance.add_state(bulk_performance.state)
 
             # memory clean
-            gc.collect()
+            gc.collect(generation = 2)
 
         return performance
 
@@ -415,7 +415,7 @@ def run_executor(self, executor_list = ExecutorHelper.PROCESS_2_8_THREAD_1_4_SHO
                         performance.add_state(state)
 
                 # memory clean
-                gc.collect(generation = 1)
+                gc.collect(generation = 2)
 
             self._print_footer(file, performance.state)
 
@@ -470,6 +470,7 @@ def run(self, processes = 2, threads = 2, run_setup: RunSetup = None, performanc
                 else:
                     performance.add_state(state)
 
+            gc.collect(generation = 2)
             self._print_footer(file, performance.state)
 
         except Exception as e:
@@ -559,15 +560,21 @@ def test_run(self, run_setup: RunSetup=None, print_output=False) -> bool:
     # region GRAPH's
 
     @staticmethod
-    def create_graph_static(input_file, output_graph_dir = "output", scope: GraphScope = GraphScope.all_no_raw, picture_dpi = 100, suppress_error = False) -> list[str]:
+    def create_graph_static(input_file,
+                            output_graph_dir = "output",
+                            scope: GraphScope = GraphScope.all_no_raw,
+                            picture_dpi = 100,
+                            suppress_error = False,
+                            only_new = False) -> list[str]:
         """
         Generate graph(s) based on output from performance tests
 
         :param input_file:          source file with detail of outputs from performance tests
         :param output_graph_dir:    directory for graph outputs (with subdirectory 'graph-perf' and 'graph-exec')
-        :param scope:               definition of scope generation (default ExecutorGraph.all)
+        :param scope:               definition of scope generation (default ExecutorGraph.all_no_raw)
         :param picture_dpi:         quality of picture (default is 100 DPI)
         :param suppress_error:      suppress error (default is False)
+        :param only_new:            generate only new outputs (default is False, regenerate all)
         :return:                    list of generated files
         """
         output_file=[]
@@ -577,15 +584,15 @@ def create_graph_static(input_file, output_graph_dir = "output", scope: GraphSco
             from qgate_graph.graph_performance import GraphPerformance
 
             # raw format False
-            graph = GraphPerformance(picture_dpi, raw_format = False)
+            graph = GraphPerformance(picture_dpi, raw_format = False, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir,"graph-perf"), suppress_error):
                 output_file.append(file)
 
         if GraphScope.perf_raw in scope:
             from qgate_graph.graph_performance import GraphPerformance
 
             # raw format True
-            graph = GraphPerformance(picture_dpi, raw_format = True)
+            graph = GraphPerformance(picture_dpi, raw_format = True, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir,"graph-perf"), suppress_error):
                 output_file.append(file)
 
@@ -594,15 +601,15 @@ def create_graph_static(input_file, output_graph_dir = "output", scope: GraphSco
             from qgate_graph.graph_performance_csv import GraphPerformanceCsv
 
             # raw format False
-            graph = GraphPerformanceCsv(raw_format = False)
+            graph = GraphPerformanceCsv(raw_format = False, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir,"graph-perf"), suppress_error):
                 output_file.append(file)
 
         if GraphScope.perf_csv_raw in scope:
             from qgate_graph.graph_performance_csv import GraphPerformanceCsv
 
             # raw format True
-            graph = GraphPerformanceCsv(raw_format = True)
+            graph = GraphPerformanceCsv(raw_format = True, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir,"graph-perf"), suppress_error):
                 output_file.append(file)
 
@@ -611,75 +618,96 @@ def create_graph_static(input_file, output_graph_dir = "output", scope: GraphSco
             from qgate_graph.graph_performance_txt import GraphPerformanceTxt
 
             # raw format False
-            graph = GraphPerformanceTxt(raw_format=False)
+            graph = GraphPerformanceTxt(raw_format=False, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir, "graph-perf"), suppress_error):
                 output_file.append(file)
 
         if GraphScope.perf_txt_raw in scope:
             from qgate_graph.graph_performance_txt import GraphPerformanceTxt
 
             # raw format True
-            graph = GraphPerformanceTxt(raw_format=True)
+            graph = GraphPerformanceTxt(raw_format=True, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir, "graph-perf"), suppress_error):
                 output_file.append(file)
 
         # EXE
         if GraphScope.exe in scope:
             from qgate_graph.graph_executor import GraphExecutor
 
-            graph = GraphExecutor(picture_dpi)
+            graph = GraphExecutor(picture_dpi, only_new = only_new)
             for file in graph.generate_from_file(input_file, os.path.join(output_graph_dir,"graph-exec"), suppress_error):
                 output_file.append(file)
 
+        # clean GC
+        gc.collect(generation = 2)
         return output_file
 
-    def create_graph(self, output_graph_dir = "output", scope: GraphScope = GraphScope.all_no_raw, picture_dpi = 100, suppress_error = False) -> list[str]:
+    def create_graph(self,
+                     output_graph_dir = "output",
+                     scope: GraphScope = GraphScope.all_no_raw,
+                     picture_dpi = 100,
+                     suppress_error = False,
+                     only_new = False) -> list[str]:
         """
         Generate graph(s) based on output from performance tests.
         The outputs will be in subdirectories 'graph-perf' and 'graph-exec'.
 
         :param output_graph_dir:    directory for graph outputs (with subdirectory 'graph-perf' and 'graph-exec')
-        :param scope:               definition of scope generation (default ExecutorGraph.all)
+        :param scope:               definition of scope generation (default ExecutorGraph.all_no_raw)
         :param picture_dpi:         quality of picture (default is 100 DPI)
         :param suppress_error:      suppress error (default is False)
+        :param only_new:            generate only new outputs (default is False, regenerate all)
         :return:                    list of generated files
         """
         return ParallelExecutor.create_graph_static(self._output_file,
-                                      output_graph_dir,
-                                      scope,
-                                      picture_dpi,
-                                      suppress_error)
-
-    def create_graph_perf(self, output_graph_dir = "output", picture_dpi = 100, suppress_error = False) -> list[str]:
+                                                    output_graph_dir,
+                                                    scope,
+                                                    picture_dpi,
+                                                    suppress_error,
+                                                    only_new)
+
+    def create_graph_perf(self,
+                          output_graph_dir = "output",
+                          picture_dpi = 100,
+                          suppress_error = False,
+                          only_new = False) -> list[str]:
         """
         Generate performance graph(s) based on output from performance tests.
         The outputs will be in subdirectory 'graph-perf'.
 
         :param output_graph_dir:    directory for graph outputs (with subdirectory 'graph-perf')
         :param picture_dpi:         quality of picture (default is 100 DPI)
         :param suppress_error:      suppress error (default is False)
+        :param only_new:            generate only new outputs (default is False, regenerate all)
         :return:                    list of generated files
         """
         return ParallelExecutor.create_graph_static(self._output_file,
-                                      output_graph_dir,
-                                      GraphScope.perf,
-                                      picture_dpi,
-                                      suppress_error)
-
-    def create_graph_exec(self, output_graph_dir = "output", picture_dpi = 100, suppress_error = False) -> list[str]:
+                                                    output_graph_dir,
+                                                    GraphScope.perf,
+                                                    picture_dpi,
+                                                    suppress_error,
+                                                    only_new)
+
+    def create_graph_exec(self,
+                          output_graph_dir = "output",
+                          picture_dpi = 100,
+                          suppress_error = False,
+                          only_new = False) -> list[str]:
         """
         Generate executors graph(s) based on output from performance tests.
         The outputs will be in subdirectory 'graph-exec'.
 
         :param output_graph_dir:    directory for graph outputs (with subdirectory 'graph-exec')
         :param picture_dpi:         quality of picture (default is 100 DPI)
         :param suppress_error:      suppress error (default is False)
+        :param only_new:            generate only new outputs (default is False, regenerate all)
         :return:                    list of generated files
         """
         return ParallelExecutor.create_graph_static(self._output_file,
-                                      output_graph_dir,
-                                      GraphScope.exe,
-                                      picture_dpi,
-                                      suppress_error)
+                                                    output_graph_dir,
+                                                    GraphScope.exe,
+                                                    picture_dpi,
+                                                    suppress_error,
+                                                    only_new)
 
     # endregion GRAPHS

qgate_perf/version.py

@@ -1,3 +1,3 @@
 # Store the version here so:
 
-__version__ = '0.4.40'
+__version__ = '0.4.41'

requirements.txt

@@ -6,4 +6,4 @@ numpy>=1.26.4,<=2.1.2
 psutil>=5.9.0,<=6.0.0
 packaging>=21.0,<=24.1
 
-qgate_graph==1.4.26
+qgate_graph==1.4.27

tests/test_graph.py

@@ -16,7 +16,7 @@ def prf_test(run_setup: RunSetup) -> ParallelProbe:
 
     generator = numpy.random.default_rng()
 
-    # init (contain executor synchonization, if needed)
+    # init (contain executor synchronization, if needed)
     probe = ParallelProbe(run_setup)
 
     if run_setup.is_init:
@@ -215,3 +215,19 @@ def test_graph_percentile(self):
         self.assertTrue(len(file)==2)
         print(file[0])
 
+    def test_graph_onlynew(self):
+
+        generator = ParallelExecutor(prf_test,
+                                     label="test_percentile",
+                                     detail_output=True,
+                                     output_file=path.join(self.OUTPUT_ADR, "only_new", "perf_test_percentile.txt"))
+
+        setup=RunSetup(duration_second=2, start_delay=0, parameters={"percentile": 0.95})
+        generator.run_bulk_executor([[1,1]], [[1,1]], setup)
+
+        output=generator.create_graph_perf(self.OUTPUT_ADR, only_new = True)
+        self.assertTrue(len(output) == 1)
+
+        output=generator.create_graph_perf(self.OUTPUT_ADR, only_new = True)
+        self.assertTrue(len(output) == 0)
+

tests/test_perf_partly.py

@@ -56,6 +56,7 @@ def prf_partly(run_setup: RunSetup) -> ParallelProbe:
 
 
 class TestCasePerfPartly(unittest.TestCase):
+    """Performance measurement in more code segments (not in one block START and STOP) """
 
     OUTPUT_ADR = "../output/test_perf/"
     @classmethod