diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4b8033d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +src/mantidprofiler.egg-info/ +src/mantidprofiler/_version.py diff --git a/README.md b/README.md index 86e6c80..4962b15 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ It monitors CPU and RAM usage, and reports the time span of each algorithm (curr To profile the `SNSPowderReduction.py` workflow: ``` -python SNSPowderReduction.py & python path/to/mantid-profiler/mantid-profiler.py $! +python SNSPowderReduction.py & python path/to/mantid-profiler/src/mantidprofiler/mantidprofiler.py $! ``` The script attaches to the last spawned process, so you can also use the profiler if you are working with `MantidPlot`: ``` diff --git a/src/mantidprofiler/algorithm_tree.py b/src/mantidprofiler/algorithm_tree.py index 55099ea..10a3706 100644 --- a/src/mantidprofiler/algorithm_tree.py +++ b/src/mantidprofiler/algorithm_tree.py @@ -134,8 +134,14 @@ def apply_multiple_trees(trees, check, func): def parseLine(line): - res = re.search("ThreadID=([0-9]*), AlgorithmName=(.*), StartTime=([0-9]*), EndTime=([0-9]*)", line) - return {"thread_id": res.group(1), "name": res.group(2), "start": int(res.group(3)), "finish": int(res.group(4))} + res = { + "thread_id": re.search("ThreadID=([0-9]*)", line)[1], + "name": re.search("AlgorithmName=([a-zA-Z]*)", line)[1], + "start": int(re.search("StartTime=([0-9]*)", line)[1]), + "finish": int(re.search("EndTime=([0-9]*)", line)[1]), + } + + return res def fromFile(fileName: Path, cleanup: bool = True): diff --git a/src/mantidprofiler/children_util.py b/src/mantidprofiler/children_util.py new file mode 100644 index 0000000..1c21ad4 --- /dev/null +++ b/src/mantidprofiler/children_util.py @@ -0,0 +1,33 @@ +# children_util.py - helper functions for dealing with child processes +# +###################################################################### + + +import psutil + + +def all_children(pr: psutil.Process) -> list[psutil.Process]: + try: + return pr.children(recursive=True) + except Exception: # noqa: BLE001 + return [] + + +def update_children(old_children: dict[int, psutil.Process], new_children: list[psutil.Process]): + new_dct = {} + for ch in new_children: + new_dct.update({ch.pid: ch}) + + todel = [] + for pid in old_children.keys(): + if pid not in new_dct.keys(): + todel.append(pid) + + for pid in todel: + del old_children[pid] + + updct = {} + for pid in new_dct.keys(): + if pid not in old_children.keys(): + updct.update({pid: new_dct[pid]}) + old_children.update(updct) diff --git a/src/mantidprofiler/diskrecord.py b/src/mantidprofiler/diskrecord.py index afb5e7a..e4da91a 100644 --- a/src/mantidprofiler/diskrecord.py +++ b/src/mantidprofiler/diskrecord.py @@ -4,6 +4,7 @@ import numpy as np import psutil +from children_util import all_children from time_util import get_current_time, get_start_time @@ -27,6 +28,10 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool disk_before = process.io_counters() + children_before = {} + for ch in all_children(process): + children_before.update({ch.pid: {"process": ch, "disk": ch.io_counters()}}) + with open(logfile, "w") as handle: # add header handle.write( @@ -72,6 +77,31 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool conversion_to_size * (disk_after.write_bytes - disk_before.write_bytes) / delta_time ) + # get information from children + children_after = {} + for ch in all_children(process): + # format the children dict + children_after.update({ch.pid: {"process": ch, "disk": ch.io_counters()}}) + + # initialize change with new child + read_char_diff = children_after[ch.pid]["disk"].read_chars + write_char_diff = children_after[ch.pid]["disk"].write_chars + read_byte_diff = children_after[ch.pid]["disk"].read_bytes + write_byte_diff = children_after[ch.pid]["disk"].write_bytes + + # subtract change from last iteration, if child already existed + if ch.pid in children_before.keys(): + read_char_diff -= children_before[ch.pid]["disk"].read_chars + write_char_diff -= children_before[ch.pid]["disk"].write_chars + read_byte_diff -= children_before[ch.pid]["disk"].read_bytes + write_byte_diff -= children_before[ch.pid]["disk"].write_bytes + + # add to totals + read_char_per_sec += conversion_to_size * read_char_diff / delta_time + write_char_per_sec += conversion_to_size * write_char_diff / delta_time + read_byte_per_sec += conversion_to_size * read_byte_diff / delta_time + write_byte_per_sec += conversion_to_size * write_byte_diff / delta_time + # write information to the log file handle.write( "{0:12.6f} {1:12.3f} {2:12.3f} {3:12.3f} {4}\n".format( @@ -85,6 +115,7 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool # copy over information to new previous disk_before = disk_after + children_before = children_after last_time = current_time except (psutil.NoSuchProcess, psutil.AccessDenied): break # all done diff --git a/src/mantidprofiler/psrecord.py b/src/mantidprofiler/psrecord.py index 90f37bf..6b1fac0 100644 --- a/src/mantidprofiler/psrecord.py +++ b/src/mantidprofiler/psrecord.py @@ -37,6 +37,7 @@ import numpy as np import psutil +from children_util import all_children, update_children from time_util import get_current_time, get_start_time @@ -53,33 +54,6 @@ def get_threads(process): return process.threads() -def all_children(pr: psutil.Process) -> list[psutil.Process]: - try: - return pr.children(recursive=True) - except Exception: # noqa: BLE001 - return [] - - -def update_children(old_children: dict[int, psutil.Process], new_children: list[psutil.Process]): - new_dct = {} - for ch in new_children: - new_dct.update({ch.pid: ch}) - - todel = [] - for pid in old_children.keys(): - if pid not in new_dct.keys(): - todel.append(pid) - - for pid in todel: - del old_children[pid] - - updct = {} - for pid in new_dct.keys(): - if pid not in old_children.keys(): - updct.update({pid: new_dct[pid]}) - old_children.update(updct) - - def monitor(pid: int, logfile: Path, interval: Optional[float]) -> None: # change None to reasonable default if interval is None: