ewm7213: Independent Parsing and Child Disk Monitoring

README.md

@@ -10,7 +10,7 @@ It monitors CPU and RAM usage, and reports the time span of each algorithm (curr
 
 To profile the `SNSPowderReduction.py` workflow:
 ```
-python SNSPowderReduction.py & python path/to/mantid-profiler/mantid-profiler.py $!
+python SNSPowderReduction.py & python path/to/mantid-profiler/src/mantidprofiler/mantidprofiler.py $!
 ```
 The script attaches to the last spawned process, so you can also use the profiler if you are working with `MantidPlot`:
 ```

src/mantidprofiler/algorithm_tree.py

@@ -134,8 +134,14 @@ def apply_multiple_trees(trees, check, func):
 
 
 def parseLine(line):
-    res = re.search("ThreadID=([0-9]*), AlgorithmName=(.*), StartTime=([0-9]*), EndTime=([0-9]*)", line)
-    return {"thread_id": res.group(1), "name": res.group(2), "start": int(res.group(3)), "finish": int(res.group(4))}
+    res = {
+        "thread_id": re.search("ThreadID=([0-9]*)", line)[1],
+        "name": re.search("AlgorithmName=([a-zA-Z]*)", line)[1],
+        "start": int(re.search("StartTime=([0-9]*)", line)[1]),
+        "finish": int(re.search("EndTime=([0-9]*)", line)[1]),
+    }
+
+    return res
 
 
 def fromFile(fileName: Path, cleanup: bool = True):

src/mantidprofiler/children_util.py

@@ -0,0 +1,33 @@
+# children_util.py - helper functions for dealing with child processes
+#
+######################################################################
+
+
+import psutil
+
+
+def all_children(pr: psutil.Process) -> list[psutil.Process]:
+    try:
+        return pr.children(recursive=True)
+    except Exception:  # noqa: BLE001
+        return []
+
+
+def update_children(old_children: dict[int, psutil.Process], new_children: list[psutil.Process]):
+    new_dct = {}
+    for ch in new_children:
+        new_dct.update({ch.pid: ch})
+
+    todel = []
+    for pid in old_children.keys():
+        if pid not in new_dct.keys():
+            todel.append(pid)
+
+    for pid in todel:
+        del old_children[pid]
+
+    updct = {}
+    for pid in new_dct.keys():
+        if pid not in old_children.keys():
+            updct.update({pid: new_dct[pid]})
+    old_children.update(updct)

src/mantidprofiler/diskrecord.py

@@ -4,6 +4,7 @@
 
 import numpy as np
 import psutil
+from children_util import all_children, update_children
 from time_util import get_current_time, get_start_time
 
 
@@ -40,6 +41,10 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool
         )
         handle.write("START_TIME: {}\n".format(starting_point))
 
+        children = {}
+        for ch in all_children(process):
+            children.update({ch.pid: {"process": ch, "disk_before": ch.io_counters}})
+
         # conversion factor of bytes per sec to Giga-bits per second - 8 bits in a byte
         conversion_to_size = 1e-9
         if not show_bytes:
@@ -72,6 +77,41 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool
                         conversion_to_size * (disk_after.write_bytes - disk_before.write_bytes) / delta_time
                     )
 
+                    # get information from children
+                    update_children(children, all_children(process))
+                    for key, child in children.items():
+                        try:
+                            child_disk_after = child["process"].io_counters()
+
+                            read_char_per_sec += (
+                                conversion_to_size
+                                * (child_disk_after.read_chars - child["disk_before"].read_chars)
+                                / delta_time
+                            )
+
+                            write_char_per_sec += (
+                                conversion_to_size
+                                * (child_disk_after.write_chars - child["disk_before"].write_chars)
+                                / delta_time
+                            )
+
+                            read_byte_per_sec += (
+                                conversion_to_size
+                                * (child_disk_after.read_bytes - child["disk_before"].read_bytes)
+                                / delta_time
+                            )
+
+                            write_byte_per_sec += (
+                                conversion_to_size
+                                * (child_disk_after.write_bytes - child["disk_before"].write_bytes)
+                                / delta_time
+                            )
+
+                            child["disk_before"] = child_disk_after
+
+                        except (psutil.NoSuchProcess, psutil.AccessDenied):
+                            continue
+
                     # write information to the log file
                     handle.write(
                         "{0:12.6f} {1:12.3f} {2:12.3f} {3:12.3f} {4}\n".format(

src/mantidprofiler/psrecord.py

@@ -37,6 +37,7 @@
 
 import numpy as np
 import psutil
+from children_util import all_children, update_children
 from time_util import get_current_time, get_start_time
 
 
@@ -53,33 +54,6 @@ def get_threads(process):
     return process.threads()
 
 
-def all_children(pr: psutil.Process) -> list[psutil.Process]:
-    try:
-        return pr.children(recursive=True)
-    except Exception:  # noqa: BLE001
-        return []
-
-
-def update_children(old_children: dict[int, psutil.Process], new_children: list[psutil.Process]):
-    new_dct = {}
-    for ch in new_children:
-        new_dct.update({ch.pid: ch})
-
-    todel = []
-    for pid in old_children.keys():
-        if pid not in new_dct.keys():
-            todel.append(pid)
-
-    for pid in todel:
-        del old_children[pid]
-
-    updct = {}
-    for pid in new_dct.keys():
-        if pid not in old_children.keys():
-            updct.update({pid: new_dct[pid]})
-    old_children.update(updct)
-
-
 def monitor(pid: int, logfile: Path, interval: Optional[float]) -> None:
     # change None to reasonable default
     if interval is None: