diff --git a/util/CPU2006/combine-exec.py b/util/CPU2006/combine-exec.py new file mode 100755 index 00000000..ff96823b --- /dev/null +++ b/util/CPU2006/combine-exec.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +from io import StringIO +import csv +import re +import sys +import argparse +from contextlib import ExitStack +from typing import Dict, Iterable, List, Tuple +from collections import Counter +from openpyxl import Workbook +from openpyxl.utils import get_column_letter + + +class DuplicateDataError(Exception): + def __init__(self, old, new, message): + self.old = old + self.new = new + self.message = message + + super().__init__(f'{message} old: {old} -> new: {new}') + + +def is_blank_row(row: List[str]) -> bool: + return not row or all(cell in ('', 'NR') for cell in row[1:]) + + +def merge_tables(str_tables: Iterable[str]) -> str: + data: Dict[str, List[List[str]]] = dict() + tables = [list(csv.reader(table.splitlines())) for table in str_tables] + + for row in tables[0]: + if row: + data.setdefault(row[0], []).append(row) + + for table in tables: + nth: Dict[str, int] = Counter() + for row in table: + if not is_blank_row(row): + index = nth[row[0]] + if row[0] in data: + if not is_blank_row(data[row[0]][index]) and data[row[0]][index] != row: + raise DuplicateDataError(data[row[0]][index], row, f'Duplicate data for {row[0]}.') + data[row[0]][index] = row + nth[row[0]] += 1 + + out = StringIO() + writer = csv.writer(out) + nth: Dict[str, int] = Counter() + for row in tables[0]: + if not row: + continue + index = nth[row[0]] + writer.writerow(data[row[0]][index]) + nth[row[0]] += 1 + + return out.getvalue() + + +_RE_FOO_RESULTS_TABLE = re.compile(r'"(?P\S+ Results) Table"') + + +def extract_tables(contents: str) -> Iterable[Tuple[str, str]]: + for m in _RE_FOO_RESULTS_TABLE.finditer(contents): + tbl_start = contents.find('\n\n', m.end()) + 1 + tbl_end = contents.find('\n\n', tbl_start) + yield (m['tbl_name'], contents[tbl_start:tbl_end]) + + +def main(files, out: str): + wb = Workbook() + files = [f.read() for f in files] + tbls = map(extract_tables, files) + for tbl_group in zip(*tbls): + assert len(set(name for name, _ in tbl_group)) == 1 + ws = wb.create_sheet(tbl_group[0][0]) + + str_tables = (tbl for _, tbl in tbl_group) + merged = merge_tables(str_tables) + for row in csv.reader(merged.splitlines()): + ws.append(row) + for i, _ in enumerate(row): + ws.column_dimensions[get_column_letter(i + 1)].bestFit = True + + wb.remove(wb.active) + wb.save(out) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Merges multiple CPU2017 exec time csv results together') + parser.add_argument('-o', '--output', required=True, help='Where to write the output file') + parser.add_argument('csvs', nargs='+', help='The files to merge') + + args = parser.parse_args() + + with ExitStack() as stack: + files = [stack.enter_context(open(f, 'r')) for f in args.csvs] + + main(files, args.output) diff --git a/util/analyze/__init__.py b/util/analyze/__init__.py index 63b66c2b..1104ae3b 100644 --- a/util/analyze/__init__.py +++ b/util/analyze/__init__.py @@ -1,4 +1,5 @@ from ._types import Logs, Benchmark, Block from ._main import parse_args from .imports import import_cpu2006, import_plaidml, import_shoc, import_utils -from ._utils import * +from . import utils, ioutils +from .utils import foreach_bench diff --git a/util/analyze/_main.py b/util/analyze/_main.py index e34f92ed..00b2d487 100644 --- a/util/analyze/_main.py +++ b/util/analyze/_main.py @@ -75,16 +75,27 @@ def parse_args(parser: argparse.ArgumentParser, *names, args=None): 'plaidml': import_plaidml.parse, 'shoc': import_shoc.parse, } - parser = FILE_PARSERS[args.benchsuite] + fileparser = FILE_PARSERS[args.benchsuite] blk_filter = block_filter(args.keep_blocks_if) if args.keep_blocks_if is not True else True args_dict = vars(args) + def parse_input(x): + if isinstance(x, str): + result = fileparser(x) + if blk_filter is not True: + result = result.keep_blocks_if(blk_filter) + return result + else: + assert isinstance(x, list) + return [parse_input(l) for l in x] + # Go through the logs inputs and parse them. for name in names: - result = parser(args_dict[name]) - if blk_filter is not True: - result = result.keep_blocks_if(blk_filter) - args_dict[name] = result + args_dict[name] = parse_input(args_dict[name]) + + if hasattr(parser, '__analyze_post_process_parse_args__'): + for argname, postprocess in getattr(parser, '__analyze_post_process_parse_args__').items(): + args_dict[argname] = postprocess(args_dict[argname]) return args diff --git a/util/analyze/_types.py b/util/analyze/_types.py index 8151bdc6..22696f78 100644 --- a/util/analyze/_types.py +++ b/util/analyze/_types.py @@ -41,6 +41,9 @@ def __iter__(self): for bench in self.benchmarks: yield from bench.blocks + def __len__(self): + return sum(len(bench) for bench in self.benchmarks) + def __repr__(self): benchmarks = ','.join(b.name for b in self.benchmarks) return f'' @@ -48,6 +51,16 @@ def __repr__(self): def keep_blocks_if(self, p): return Logs([bench.keep_blocks_if(p) for bench in self.benchmarks]) + def find_equiv(self, blk): + uid = blk.uniqueid() + return [b for b in self.benchmark(blk.info['benchmark']) if b.uniqueid() == uid] + + def find_block(self, name, benchmark=None): + search = self + if benchmark is not None: + search = self.benchmark(benchmark) + return [b for b in search if b.name == name] + class Benchmark: ''' @@ -67,6 +80,9 @@ def __init__(self, info, blocks): def __iter__(self): return iter(self.blocks) + def __len__(self): + return len(self.blocks) + @property def benchmarks(self): return (self,) @@ -77,6 +93,16 @@ def __repr__(self): def keep_blocks_if(self, p): return Benchmark(self.info, [blk for blk in self.blocks if p(blk)]) + def find_equiv(self, blk): + uid = blk.uniqueid() + return [b for b in self if b.uniqueid() == uid] + + def find_block(self, name, benchmark=None): + if benchmark is not None: + if benchmark != self.name: + return [] + return [b for b in self if b.name == name] + class Block: ''' @@ -93,10 +119,14 @@ class Block: def __init__(self, info, raw_log, events): self.name = info['name'] + self.benchmark = info['benchmark'] self.info = info self.raw_log = raw_log self.events = events + if 'PassFinished' in self: + self.info['pass'] = self.single('PassFinished')['num'] + def single(self, event_name): ''' Gets an event with the specified name, requiring exactly one match @@ -132,3 +162,6 @@ def __repr__(self): def uniqueid(self): return frozenset(self.info.items()) + + def dump(self): + print(self.raw_log) diff --git a/util/analyze/_utils.py b/util/analyze/_utils.py deleted file mode 100644 index 0ff6615a..00000000 --- a/util/analyze/_utils.py +++ /dev/null @@ -1,39 +0,0 @@ -from ._types import * - - -def sum_dicts(ds): - ''' - Sums ds[N]['Key'] for each key for each dict. Assumes each dict has the same keys - E.g. sum_dicts({'a': 1, 'b': 2}, {'a': 2, 'b': 3}) produces {'a': 3, 'b': 5} - ''' - if not ds: - return {} - return {k: sum(d[k] for d in ds) for k in ds[0].keys()} - - -def foreach_bench(analysis_f, *logs, combine=None): - ''' - Repeats `analysis_f` for each benchmark in `logs`. - Also computes the analysis for the entire thing. - If `combine` is given, uses the function to combine it. - Otherwise, runs `analysis_f` over the entire thing (takes quite some time) - - Returns: - A dictionary containing the per-benchmark results. - The keys are the benchmark names. - The run for the entire thing has a key of 'Total' - ''' - - if combine is None: - combine = lambda *args: analysis_f(*logs) - - benchmarks = zip(*[log.benchmarks for log in logs]) - - bench_stats = {bench[0].name: analysis_f(*bench) for bench in benchmarks} - total = combine(bench_stats.values()) - - return { - # Making a new dict so that the "Total" key can be first. - 'Total': total, - **bench_stats, - } diff --git a/util/analyze/imports/import_cpu2006.py b/util/analyze/imports/import_cpu2006.py index c8b18363..23edd8a3 100755 --- a/util/analyze/imports/import_cpu2006.py +++ b/util/analyze/imports/import_cpu2006.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re from . import import_utils @@ -13,8 +12,8 @@ def parse(file): with open(file, 'r') as f: return import_utils.parse_multi_bench_file( f.read(), - benchstart=re.compile(r'Building (?P\S*)'), - filename=re.compile(r'/[fc]lang\b.*\s(\S+\.\S+)\n')) + benchstart=r'Building (?P\S*)', + filename=r'/[fc]lang\b.*\s(\S+\.\S+)\n') if __name__ == '__main__': diff --git a/util/analyze/imports/import_utils.py b/util/analyze/imports/import_utils.py index 3454cb67..5b5baa22 100644 --- a/util/analyze/imports/import_utils.py +++ b/util/analyze/imports/import_utils.py @@ -1,13 +1,15 @@ -import pickle -import json import itertools +import json +import pickle import re import sys -from collections import namedtuple +from dataclasses import dataclass +from typing import List, Match, Optional, Pattern, Union -from .._types import Logs, Benchmark, Block +from .._types import Benchmark, Block, Logs -_RE_REGION_INFO = re.compile(r'EVENT:.*ProcessDag.*"name": "(?P[^"]*)"') +_REGION_DELIMITER = 'INFO: ********** Opt Scheduling **********' +_RE_REGION_DELIMITER = re.compile(re.escape(_REGION_DELIMITER)) def import_main(parsefn, *, description): @@ -24,18 +26,39 @@ def import_main(parsefn, *, description): pickle.dump(result, f) -def parse_multi_bench_file(logtext, *, benchstart, filename=None): +def parse_multi_bench_file(logtext: str, *, benchstart: Union[Pattern, str], filename: Optional[Union[Pattern, str]] = None): + if filename is not None: + filename = re.compile(filename) + benchstart = re.compile(benchstart) + + def parse_bench(benchm: Match, nextm: Union[Match, _DummyEnd], is_first: bool = False): + # The RE can specify any extra properties. + info = benchm.groupdict() + # If this is the first benchmark in the file, we want to start from the + # start of the file so that we don't lose any information. + start = 0 if is_first else benchm.start() + end = nextm.start() + return _parse_benchmark(info, logtext, + start, end, + filenamere=filename) + + bench_matches = list(benchstart.finditer(logtext)) benchmarks = [] - for benchm, nextm in _splititer(benchstart, logtext): - bench = _parse_benchmark(benchm.groupdict(), logtext, - benchm.end(), nextm.start(), - filenamere=filename) - benchmarks.append(bench) + + is_first: bool = True + for benchm, nextm in zip( + bench_matches, + [*bench_matches[1:], _DummyEnd(len(logtext))] + ): + benchmarks.append(parse_bench(benchm, nextm, is_first)) + is_first = False return Logs(benchmarks) -def parse_single_bench_file(logtext, *, benchname, filename=None): +def parse_single_bench_file(logtext, *, benchname, filename: Optional[Union[Pattern, str]] = None): + if filename is not None: + filename = re.compile(filename) return Logs([ _parse_benchmark( {'name': benchname}, @@ -45,21 +68,10 @@ def parse_single_bench_file(logtext, *, benchname, filename=None): ]) -_FileInfo = namedtuple('_FileInfo', ('filename', 'from_pos')) - - -def _each_cons(iterable, n): - ''' - Iterates over each consecutive n items of the iterable. - - _each_cons((1, 2, 3, 4), 2) # (1, 2), (2, 3), (3, 4) - ''' - iters = [None] * n - iters[0] = iter(iterable) - for i in range(1, n): - iters[i - 1], iters[i] = itertools.tee(iters[i - 1]) - next(iters[i], None) - return zip(*iters) +@dataclass +class _FileInfo: + filename: Optional[str] + from_pos: int class _DummyEnd: @@ -73,58 +85,59 @@ def end(self): return self._end -def _splititer(regex, text, pos=0, endpos=None): - ''' - 'Splits' the string by the regular expression, using an iterable. - Returns both where the regex matches and where it matched next (or the end). - ''' - if endpos is None: - endpos = len(text) - 1 +def _filename_info(filenamere: Optional[Pattern], logtext: str, start: int, end: int) -> List[_FileInfo]: + if filenamere is None: + filenamere = re.compile(r'.^') # RE that doesn't match anything + files = [] - return _each_cons( - itertools.chain(regex.finditer(text, pos, endpos), - (_DummyEnd(endpos + 1),)), - 2 - ) + for filem in filenamere.finditer(logtext, start, end): + filename = filem.group(1) + filestart = filem.end() + files.append(_FileInfo(filename=filename, from_pos=filestart)) + return files -def _parse_benchmark(info, logtext: str, start, end, *, filenamere): - NAME = info['name'] + +def _parse_benchmark(info: dict, logtext: str, start: int, end: int, *, filenamere: Optional[Pattern]): + BENCHNAME = info['name'] blocks = [] - if filenamere and filenamere.search(logtext, start, end): - files = [ - *(_FileInfo(filename=r.group(1), from_pos=r.end()) - for r in filenamere.finditer(logtext, start, end)), - _FileInfo(filename=None, from_pos=len(logtext)), - ][::-1] - else: - files = [ - _FileInfo(filename=None, from_pos=start), - _FileInfo(filename=None, from_pos=len(logtext)), - ][::-1] + files: List[_FileInfo] = _filename_info(filenamere, logtext, start, end) + if not files: + # We have an unknown file starting from the very beginning + files = [_FileInfo(filename=None, from_pos=start)] + + # Allow us to peek ahead by giving a dummy "file" at the end which will never match a block + files.append(_FileInfo(filename=None, from_pos=end)) + assert len(files) >= 2 + file_pos = 0 + + block_matches1, block_matches2 = itertools.tee(_RE_REGION_DELIMITER.finditer(logtext, start, end)) + next(block_matches2) # Drop first + block_matches2 = itertools.chain(block_matches2, (_DummyEnd(end),)) blocks = [] - for regionm, nextm in _splititer(_RE_REGION_INFO, logtext, start, end): - assert regionm.end() > files[-1].from_pos - if regionm.end() > files[-2].from_pos: - files.pop() + is_first = True + for regionm, nextm in zip(block_matches1, block_matches2): + region_start = regionm.end() + if region_start > files[file_pos + 1].from_pos: + file_pos += 1 + + assert region_start > files[file_pos].from_pos - try: - filename = files[-1].filename - except NameError: - filename = None + filename = files[file_pos].filename if files[file_pos] else None regioninfo = { - 'name': regionm['name'], 'file': filename, - 'benchmark': NAME, + 'benchmark': BENCHNAME, } - block = _parse_block(regioninfo, logtext, - regionm.start() - 1, nextm.start()) - blocks.append(block) + blk_start = start if is_first else regionm.start() + blk_end = nextm.start() + blocks.append(_parse_block(regioninfo, logtext, + blk_start, blk_end)) + is_first = False return Benchmark(info, blocks) @@ -132,6 +145,8 @@ def _parse_benchmark(info, logtext: str, start, end, *, filenamere): def _parse_block(info, logtext: str, start, end): events = _parse_events(logtext, start, end) raw_log = logtext[start:end] + assert 'ProcessDag' in events + info['name'] = events['ProcessDag'][0]['name'] return Block(info, raw_log, events) diff --git a/util/analyze/ioutils.py b/util/analyze/ioutils.py new file mode 100644 index 00000000..ca52b9ec --- /dev/null +++ b/util/analyze/ioutils.py @@ -0,0 +1,95 @@ +import argparse +import csv +from io import StringIO + + +class _Writer: + def __init__(self, add_bench): + self.__add_bench = add_bench + + def __addinfo(self, bench, data): + if self.__add_bench: + return {'Benchmark': bench, **data} + return data + + def benchdata(self, bench, data): + self._benchdata(self.__addinfo(bench, data)) + + def finish(self): + self._finish() + + +class _CSVWriter(_Writer): + def __init__(self, f, data: dict, fieldnames=None): + add_bench = fieldnames is None or 'Benchmark' in fieldnames and 'Benchmark' not in data + super().__init__(add_bench) + + if fieldnames is None: + fieldnames = ['Benchmark', *data['Total'].keys()] + + self.__f = f + self.__mem_file = StringIO() + self.__csv_writer = csv.DictWriter(self.__mem_file, fieldnames=fieldnames) + self.__csv_writer.writeheader() + + def _benchdata(self, data): + self.__csv_writer.writerow(data) + + def _finish(self): + self.__mem_file.seek(0) + transposed = zip(*csv.reader(self.__mem_file)) + csv.writer(self.__f).writerows(transposed) + + +class _HumanWriter(_Writer): + def __init__(self, f, data: dict, fieldnames=None): + add_bench = fieldnames is None or 'Benchmark' in fieldnames and 'Benchmark' not in data + super().__init__(add_bench) + + if fieldnames is None: + fieldnames = ['Benchmark', *data['Total'].keys()] + + self.__f = f + self.__fieldnames = fieldnames + self.__data = {name: [f'{name}:'] for name in fieldnames} + self.__num_entries = 1 + + def _benchdata(self, data): + self.__num_entries += 1 + for k, v in data.items(): + self.__data[k].append(str(v)) + + def _finish(self): + col_max = [max(len(self.__data[field][index]) for field in self.__fieldnames) + for index in range(self.__num_entries)] + for field in self.__fieldnames: + for index, val in enumerate(self.__data[field]): + self.__f.write(f'{val:{col_max[index]+1}}') + self.__f.write('\n') + + +def _write_data(writer: _Writer, data: dict): + for bench, bench_data in data.items(): + writer.benchdata(bench, bench_data) + writer.finish() + + +def write_csv(f, data: dict, *, fieldnames=None): + _write_data(_CSVWriter(f, data, fieldnames), data) + + +def write_human(f, data: dict, *, fieldnames=None): + _write_data(_HumanWriter(f, data, fieldnames), data) + + +def add_output_format_arg(parser: argparse.ArgumentParser, default='csv'): + parser.add_argument('--format', default=default, choices=('csv', 'human'), + help=f'Which format style to use (default: {default})') + FORMAT_OPTIONS = { + 'csv': write_csv, + 'human': write_human, + } + + if not hasattr(parser, '__analyze_post_process_parse_args__'): + setattr(parser, '__analyze_post_process_parse_args__', {}) + getattr(parser, '__analyze_post_process_parse_args__')['format'] = FORMAT_OPTIONS.__getitem__ diff --git a/util/analyze/lib/block_stats.py b/util/analyze/lib/block_stats.py new file mode 100755 index 00000000..cfbc3814 --- /dev/null +++ b/util/analyze/lib/block_stats.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +from typing import * +import argparse +from analyze import ioutils +import analyze +from analyze import Block, Logs, utils + + +def is_enumerated(blk: Block) -> bool: + return 'Enumerating' in blk + + +def is_optimal(blk: Block) -> bool: + return 'DagSolvedOptimally' in blk or 'HeuristicScheduleOptimal' in blk + + +def is_timed_out(blk: Block) -> bool: + return 'DagTimedOut' in blk + + +def block_cost_lower_bound(blk: Block) -> int: + return blk['CostLowerBound'][-1]['cost'] + + +def block_relative_cost(blk: Block) -> int: + return blk.single('BestResult')['cost'] + + +def block_best_length(blk: Block) -> int: + return blk.single('BestResult')['length'] + + +def block_cost(blk: Block) -> int: + return block_cost_lower_bound(blk) + block_relative_cost(blk) + + +def cost_improvement_for_blk(blk: Block) -> int: + if 'DagSolvedOptimally' in blk: + return blk.single('DagSolvedOptimally')['cost_improvement'] + elif 'DagTimedOut' in blk: + return blk.single('DagTimedOut')['cost_improvement'] + else: + return 0 + + +def is_improved(blk: Block) -> bool: + return cost_improvement_for_blk(blk) > 0 + + +def nodes_examined_for_blk(blk: Block) -> int: + return blk.single('NodeExamineCount')['num_nodes'] if 'NodeExamineCount' in blk else 0 + + +def num_blocks(logs: Logs) -> int: + return sum(len(bench.blocks) for bench in logs.benchmarks) + + +def num_enumerated(logs: Logs) -> int: + return sum(1 for blk in logs if is_enumerated(blk)) + + +def nodes_examined(logs: Logs) -> int: + return sum(nodes_examined_for_blk(blk) for blk in logs) + + +def compute_block_stats(logs: Logs): + return { + 'num blocks': num_blocks(logs), + 'num blocks enumerated': num_enumerated(logs), + 'num optimal and improved': utils.count(blk for blk in logs if is_optimal(blk) and is_improved(blk) and is_enumerated(blk)), + 'num optimal and not improved': utils.count(blk for blk in logs if is_optimal(blk) and not is_improved(blk) and is_enumerated(blk)), + 'num not optimal and improved': utils.count(blk for blk in logs if not is_optimal(blk) and is_improved(blk) and is_enumerated(blk)), + 'num not optimal and not improved': utils.count(blk for blk in logs if not is_optimal(blk) and not is_improved(blk) and is_enumerated(blk)), + 'nodes examined': nodes_examined(logs), + } + + +if __name__ == '__main__': + import sys + import csv + + parser = argparse.ArgumentParser( + description='Computes the block stats for the logs') + parser.add_argument('logs', help='The logs to analyze') + ioutils.add_output_format_arg(parser) + args = analyze.parse_args(parser, 'logs') + + results = utils.foreach_bench(compute_block_stats, args.logs) + + args.format(results) diff --git a/util/analyze/lib/compile_times.py b/util/analyze/lib/compile_times.py index 312a96ab..25ac986d 100755 --- a/util/analyze/lib/compile_times.py +++ b/util/analyze/lib/compile_times.py @@ -4,6 +4,7 @@ import re import argparse import sys +import logging import analyze from analyze import Block, foreach_bench @@ -15,27 +16,98 @@ def _block_time(block: Block): return end - start -def instruction_scheduling_time(logs): +def sched_time(logs): return sum(_block_time(blk) for blk in logs) +def heuristic_time_for_blk(blk: Block) -> int: + return blk.single('HeuristicResult')['elapsed'] + + +def heuristic_time(logs): + return sum(heuristic_time_for_blk(b) for b in logs) + + +def first_heuristic_time_for_blk(blk: Block) -> int: + return blk['HeuristicResult'][0]['elapsed'] + + +def first_heuristic_time(logs): + return sum(first_heuristic_time_for_blk(b) for b in logs) + + +def first_lower_bound_time_for_blk(blk: Block) -> int: + return blk['CostLowerBound'][0]['elapsed'] + + +def first_lower_bound_time(logs): + return sum(first_lower_bound_time_for_blk(blk) for blk in logs) + + +_CPU2017_TIME_ELAPSED = re.compile(r"Elapsed compile for '(?P[^']+)': \S+ \((?P\d+)\)") +_BACKUP_TIME_ELAPSED = re.compile(r'(?P\d+) total seconds elapsed') +_PLAIDML_TIME_ELAPSED = re.compile( + r'Example finished, elapsed: (?P\S+)s \(compile\), (?P\S+)s \(execution\)') +_SHOC_TIME_ELAPSED = re.compile(r'Finished compiling; total ns = (?P\d+)') + + +def shoc_total_compile_time_seconds(logs): + try: + elapsed = sum(int(m['elapsed']) + for bench in logs.benchmarks + for blk in bench + for m in _SHOC_TIME_ELAPSED.finditer(blk.raw_log)) + return float(elapsed) * 1e-9 + except TypeError: + raise KeyError('Logs must contain "Finished compiling; total ns = " output by the modified SHOC benchmark suite') + + +def plaidml_total_compile_time_seconds(logs): + try: + return sum(float(_PLAIDML_TIME_ELAPSED.search(bench.blocks[-1].raw_log)['elapsed']) for bench in logs.benchmarks) + except TypeError: + raise KeyError('Logs must contain "Example finished, elapsed:" output by the PlaidML benchmark suite') + + def total_compile_time_seconds(logs): - last_logs = logs.benchmarks[-1].blocks[-1].raw_log - m = re.search(r'(\d+) total seconds elapsed', last_logs) + last_blk = logs.benchmarks[-1].blocks[-1] + last_logs = last_blk.raw_log + m = [g for g in _CPU2017_TIME_ELAPSED.finditer(last_logs) + if last_blk.benchmark == g['bench']] + + if m: + if len(m) != 1: + logging.warning('Multiple CPU2017 elapsed time indicators. Using the first one out of: %s', m) + return int(m[0]['elapsed']) + + m = _BACKUP_TIME_ELAPSED.search(last_logs) assert m, \ 'Logs must contain "total seconds elapsed" output by the SPEC benchmark suite' - return m.group(1) + return int(m['elapsed']) + + +def total_compile_time_seconds_f(benchsuite): + return { + 'spec': total_compile_time_seconds, + 'plaidml': plaidml_total_compile_time_seconds, + 'shoc': shoc_total_compile_time_seconds, + }[benchsuite] if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--variant', choices=('sched', 'total'), + parser.add_argument('--variant', choices=('sched', 'total', 'plaidml'), help='Which timing variant to use') parser.add_argument('logs', help='The logs to analyze') args = analyze.parse_args(parser, 'logs') - fn = total_compile_time_seconds if args.variant == 'total' else instruction_scheduling_time + fn = { + 'sched': sched_time, + 'total': total_compile_time_seconds, + 'plaidml': plaidml_total_compile_time_seconds, + 'shoc': shoc_total_compile_time_seconds, + }[args.variant] results = foreach_bench(fn, args.logs, combine=sum) writer = csv.DictWriter(sys.stdout, fieldnames=results.keys()) writer.writeheader() diff --git a/util/analyze/lib/find_negative_nodes_examined.py b/util/analyze/lib/find_negative_nodes_examined.py new file mode 100755 index 00000000..c72957ee --- /dev/null +++ b/util/analyze/lib/find_negative_nodes_examined.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +from typing import * +import argparse +import analyze +from analyze import Block, Logs + + +def nodes_examined(block: Block) -> int: + return block.single('NodeExamineCount')['num_nodes'] if 'NodeExamineCount' in block else 0 + + +def is_negative_nodes_examined(first: Block, second: Block) -> bool: + return nodes_examined(first) < nodes_examined(second) + + +def find_negative_nodes_examined(first: Logs, second: Logs, percent_threshold: float = 0, absolute_threshold: float = 0) -> List[Tuple[Block, Block, int, int]]: + return [ + (f, s, nodes_examined(f), nodes_examined(s)) for f, s in zip(first, second) + if is_negative_nodes_examined(f, s) + and nodes_examined(s) - nodes_examined(f) < absolute_threshold + and (nodes_examined(s) - nodes_examined(f)) / nodes_examined(f) * 100 < percent_threshold + ] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Finds all blocks for which nodes_examined(first_logs) < nodes_examined(second_logs)') + parser.add_argument('first', help='The first logs') + parser.add_argument('second', help='The second logs') + parser.add_argument('-%', '--percent-threshold', type=float, default=0, + help='Ignore any blocks with a %%-difference < threshold') + parser.add_argument('-$', '--absolute-threshold', type=float, default=0, + help='Ignore any blocks with a difference < threshold') + args = analyze.parse_args(parser, 'first', 'second') + + negatives = find_negative_nodes_examined( + args.first, args.second, percent_threshold=args.percent_threshold, absolute_threshold=args.absolute_threshold) + negatives = sorted(negatives, key=lambda x: x[3] - x[2]) + for fblock, sblock, num_f, num_s in negatives: + print( + f"{fblock.info['benchmark']} {fblock.name} : {num_f} - {num_s} = {num_f - num_s}") diff --git a/util/analyze/lib/func_stats.py b/util/analyze/lib/func_stats.py new file mode 100755 index 00000000..43b3c2d6 --- /dev/null +++ b/util/analyze/lib/func_stats.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +import argparse +import re +import sys +from itertools import chain +from typing import Callable, Iterable, List, Pattern, Tuple + +import analyze +from analyze import Block, ioutils, utils + +''' +Function-level stats (not Block, Logs, or Benchmark level) +''' + +_RE_OCCUPANCY = re.compile(r'Final occupancy for function (?P\S+):(?P\d+)') +_RE_SPILLS = re.compile(r'Function: (?P\S*?)\nGREEDY RA: Number of spilled live ranges: (?P\d+)') +_RE_SPILLS_WEIGHTED = re.compile(r'SC in Function (?P\S*?) (?P-?\d+)') + + +def compute_avg_values(fn_info: List[Tuple[str, int]], *, fn_filter: Callable[[str, int], bool] = lambda k, v: True) -> float: + return utils.average((v for k, v in fn_info if fn_filter(k, v)), len(fn_info)) + + +def _fn_re_info(re: Pattern, logs: Iterable[Block], key='name', value='value') -> Iterable[Tuple[str, int]]: + for m in chain.from_iterable(re.finditer(blk.raw_log) for blk in logs): + yield (m[key], int(m[value])) + + +def fn_occupancy_info(logs: Iterable[Block]) -> List[Tuple[str, int]]: + return list(_fn_re_info(_RE_OCCUPANCY, logs)) + + +def avg_occupancy(logs: Iterable[Block], *, fn_filter: Callable[[str, int], bool] = lambda k, v: True) -> float: + occ_info = fn_occupancy_info(logs) + return compute_avg_values(occ_info, fn_filter=fn_filter) + + +def fn_spill_info(logs: Iterable[Block]) -> List[Tuple[str, int]]: + return list(_fn_re_info(_RE_SPILLS, logs)) + + +def fn_weighted_spill_info(logs: Iterable[Block]) -> List[Tuple[str, int]]: + return list(_fn_re_info(_RE_SPILLS_WEIGHTED, logs)) + + +def total_spills(logs: Iterable[Block], *, fn_filter: Callable[[str, int], bool] = lambda k, v: True) -> int: + return sum(v for k, v in fn_spill_info(logs) if fn_filter(k, v)) + + +def total_weighted_spills(logs: Iterable[Block], *, fn_filter: Callable[[str, int], bool] = lambda k, v: True) -> int: + return sum(v for k, v in fn_weighted_spill_info(logs) if fn_filter(k, v)) + + +def raw_main(argv: List[str] = []): + parser = argparse.ArgumentParser( + description='Computes the block stats for the logs') + parser.add_argument('--stat', required=True, choices=('occ', 'spills', 'weighted-spills'), + help='Which stat to compute') + parser.add_argument('--hot-only', help='A file with a space-separated list of functions to consider in the count') + ioutils.add_output_format_arg(parser) + parser.add_argument('logs', help='The logs to analyze') + args = analyze.parse_args(parser, 'logs', args=argv) + + if args.hot_only: + with open(args.hot_only, 'r') as f: + contents = f.read() + fns = set(contents.split()) + def fn_filter(k, v): return k in fns + else: + def fn_filter(k, v): return True + + STATS = { + 'occ': ('Average Occupancy', avg_occupancy), + 'spills': ('Spill Count', total_spills), + 'weighted-spills': ('Weighted Spill Count', total_weighted_spills), + } + label, f = STATS[args.stat] + + results = utils.foreach_bench(lambda bench: {label: f(bench, fn_filter=fn_filter)}, args.logs) + + args.format(sys.stdout, results) + + +if __name__ == '__main__': + raw_main(None) # Default to sys.argv diff --git a/util/analyze/utils.py b/util/analyze/utils.py new file mode 100644 index 00000000..b92355f0 --- /dev/null +++ b/util/analyze/utils.py @@ -0,0 +1,130 @@ +from typing import Iterable +from ._types import * + + +def sum_dicts(ds): + ''' + Sums ds[N]['Key'] for each key for each dict. Assumes each dict has the same keys + E.g. sum_dicts({'a': 1, 'b': 2}, {'a': 2, 'b': 3}) produces {'a': 3, 'b': 5} + ''' + if not ds: + return {} + return {k: sum(d[k] for d in ds) for k in ds[0].keys()} + + +def foreach_bench(analysis_f, *logs, combine=None, **kwargs): + ''' + Repeats `analysis_f` for each benchmark in `logs`. + Also computes the analysis for the entire thing. + If `combine` is given, uses the function to combine it. + Otherwise, runs `analysis_f` over the entire thing (takes quite some time) + + Returns: + A dictionary containing the per-benchmark results. + The keys are the benchmark names. + The run for the entire thing has a key of 'Total' + ''' + + if combine is None: + combine = lambda *args: analysis_f(*logs, **kwargs) + + benchmarks = zip(*[log.benchmarks for log in logs]) + + bench_stats = {bench[0].name: analysis_f(*bench, **kwargs) for bench in benchmarks} + total = combine(bench_stats.values()) + + return { + # Making a new dict so that the "Total" key can be first. + 'Total': total, + **bench_stats, + } + + +def count(iter): + try: + return len(iter) + except: + return sum(1 for _ in iter) + + +def zipped_keep_blocks_if(*logs, pred): + ''' + Given: + a: [blk1, blk2, blk3, ...] # of type Logs + b: [blk1, blk2, blk3, ...] # of type Logs + c: [blk1, blk2, blk3, ...] # of type Logs + ... + + Returns: + [ + (a.blk1, b.blk1, c.blk1, ...) if pred(a.blk1, b.blk1, c.blk1, ...) + ... + ] + + Also supports pred(b), in which case it's all(pred(b) for b in (a.blk1, b.blk1, ...)) + ''' + if not logs: + return [] + + for group in zip(*logs): + assert len(set(g.uniqueid() for g in group)) == 1, group[0].raw_log + + try: + blks = next(zip(*logs)) + pred(*blks) + except TypeError: + old_pred = pred + pred = lambda *blks: all(old_pred(b) for b in blks) + except StopIteration: + # There was nothing in zip(*logs)... + old_pred = pred + + def new_pred(*blks): + try: + return old_pred(*blks) + except TypeError: + return all(old_pred(b) for b in blks) + pred = new_pred + + def zip_benchmarks_if(*benchmarks): + # (A[a], A[a]) -> [(a, a)] or [] + zipped = [blks for blks in zip(*benchmarks) if pred(*blks)] + unzipped = list(zip(*zipped)) # [(a, a)] -> ([a], [a]); [] -> [] + if not unzipped: # if []: ([], []) + unzipped = [()] * len(benchmarks) + # ([a], [a]) -> (A[a], A[a]) + return [Benchmark(bench.info, bench_blks) for bench, bench_blks in zip(benchmarks, unzipped)] + + # [(Bench.X, Bench.X), (Bench.Y, Bench.Y)] + result = [] + + # L1: [A[a], B[b], C[c]] + # L2: [A[a], B[b], C[c]] + # benchs: [(A[a], A[a]), (B[b], B[b]), ...] + benchs = zip(*[l.benchmarks for l in logs]) + # filtered_benchs: [(A[a], A[a]), (B[b], B[b]), ...] + filtered_benchs = (zip_benchmarks_if(*bench_grp) for bench_grp in benchs) + # [(A[a], A[a]), (B[b], B[b])] -> ([A[a], B[b], ...], [A[a], B[b], ...]) + log_benchs = zip(*filtered_benchs) + new_logs = map(Logs, log_benchs) + + return tuple(new_logs) + + +def sum_stat_for_all(stat, logs: Logs) -> int: + return sum(stat(blk) for blk in logs) + + +def average(xs: Iterable[int], count=None) -> float: + try: + size = count if count is not None else len(xs) + return sum(xs) / size if size else 0.0 + except TypeError: + pass + + acc = 0 + num = 0 + for x in xs: + acc += x + num += 1 + return acc / num if num else 0.0 diff --git a/util/gt_analysis/__init__.py b/util/gt_analysis/__init__.py new file mode 100644 index 00000000..115122e5 --- /dev/null +++ b/util/gt_analysis/__init__.py @@ -0,0 +1,2 @@ +from . import gt_cmp +from . import gt_cmp_opt_only diff --git a/util/gt_analysis/gt_cmp.py b/util/gt_analysis/gt_cmp.py new file mode 100755 index 00000000..3077b80b --- /dev/null +++ b/util/gt_analysis/gt_cmp.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + +import argparse +from typing import Tuple + +import analyze +from analyze import Block, Logs, utils, ioutils +from analyze.lib import block_stats, compile_times + +sched_time = compile_times.sched_time + + +def blocks_enumerated_optimally(blocks): + return [blk for blk in blocks if 'DagSolvedOptimally' in blk or 'HeuristicScheduleOptimal' in blk] + + +def rp_ilp_gt_elapsed_for_blk(blk: Block) -> int: + if 'GraphTransOccupancyPreservingILPNodeSuperiority' not in blk: + return 0 + return blk.single('GraphTransOccupancyPreservingILPNodeSuperiorityFinished')['time'] \ + - blk.single('GraphTransOccupancyPreservingILPNodeSuperiority')['time'] + + +def rp_only_gt_elapsed_for_blk(blk: Block) -> int: + if 'GraphTransRPNodeSuperiority' not in blk: + return 0 + return blk.single('GraphTransRPNodeSuperiorityFinished')['time'] \ + - blk.single('GraphTransRPNodeSuperiority')['time'] + + +def ilp_only_gt_elapsed_for_blk(blk: Block) -> int: + if 'GraphTransILPNodeSuperiority' not in blk: + return 0 + return blk.single('GraphTransILPNodeSuperiorityFinished')['time'] \ + - blk.single('GraphTransILPNodeSuperiority')['time'] + + +def raw_gt_elapsed_for_blk(blk: Block) -> int: + return rp_ilp_gt_elapsed_for_blk(blk) \ + + rp_only_gt_elapsed_for_blk(blk) \ + + ilp_only_gt_elapsed_for_blk(blk) + + +def total_gt_elapsed_for_blk(blk: Block) -> int: + if 'GraphTransformationsStart' not in blk: + return 0 + return blk.single('GraphTransformationsFinished')['time'] \ + - blk.single('GraphTransformationsStart')['time'] + + +def elapsed_before_enumeration_for_blk(blk: Block) -> int: + assert 'CostLowerBound' in blk + return blk['CostLowerBound'][-1]['time'] + + +def enum_time_for_blk(blk: Block) -> int: + if 'DagSolvedOptimally' not in blk: + return blk.single('DagTimedOut')['time'] - blk['Enumerating'][0]['time'] + return blk.single('DagSolvedOptimally')['time'] - blk['Enumerating'][0]['time'] + + +def cost_for_blk(blk: Block) -> int: + return blk.single('BestResult')['cost'] + blk['CostLowerBound'][-1]['cost'] + + +def is_improved(before: Block, after: Block): + return cost_for_blk(before) > cost_for_blk(after) + + +def blk_relative_cost(nogt, gt) -> Tuple[int, int]: + no_sum = yes_sum = 0 + for no, yes in zip(nogt, gt): + no_cost = block_stats.block_cost(no) + yes_cost = block_stats.block_cost(yes) + no_lb = block_stats.block_cost_lower_bound(no) + yes_lb = block_stats.block_cost_lower_bound(yes) + assert no_lb <= yes_lb + + # relative to the tightest LB we know + no_rel = no_cost - yes_lb + yes_rel = yes_cost - yes_lb + + no_sum += no_rel + yes_sum += yes_rel + + return no_sum, yes_sum + + +RP_GT_FINISHED = 'GraphTransRPNodeSuperiorityFinished' +ILP_GT_FINISHED = 'GraphTransILPNodeSuperiorityFinished' +RP_ILP_GT_FINISHED = 'GraphTransOccupancyPreservingILPNodeSuperiorityFinished' + + +def _edges_added_for_blk(blk: Block, fin_id: str) -> int: + if fin_id not in blk: + return 0 + return sum(ev['superior_edges'] for ev in blk[fin_id]) + + +def edges_added_for_blk(blk: Block) -> int: + return _edges_added_for_blk(blk, RP_GT_FINISHED) + _edges_added_for_blk(blk, ILP_GT_FINISHED) + _edges_added_for_blk(blk, RP_ILP_GT_FINISHED) + + +def _edges_removed_for_blk(blk: Block, fin_id: str) -> int: + if fin_id not in blk: + return 0 + try: + return sum(ev['removed_edges'] for ev in blk[fin_id]) + except KeyError: + return 0 + + +def edges_removed_for_blk(blk: Block) -> int: + return _edges_removed_for_blk(blk, RP_GT_FINISHED) + _edges_removed_for_blk(blk, ILP_GT_FINISHED) + _edges_removed_for_blk(blk, RP_ILP_GT_FINISHED) + + +def edges_rp_rejected_for_blk(blk: Block) -> int: + if RP_ILP_GT_FINISHED not in blk: + return 0 + return sum(ev['failed_rp'] for ev in blk[RP_ILP_GT_FINISHED]) + + +def compute_stats(nogt: Logs, gt: Logs, *, pass_num: int, total_compile_time_seconds): + nogt_all, gt_all = nogt, gt + + if pass_num is not None: + nogt = nogt.keep_blocks_if(lambda b: b.single('PassFinished')['num'] == pass_num) + gt = gt.keep_blocks_if(lambda b: b.single('PassFinished')['num'] == pass_num) + + NUM_PROVED_OPTIMAL_WITHOUT_ENUMERATING = utils.count(utils.zipped_keep_blocks_if( + nogt, gt, pred=lambda a, b: block_stats.is_enumerated(a) and not block_stats.is_enumerated(b))[0]) + nogt, gt = utils.zipped_keep_blocks_if( + nogt, gt, pred=block_stats.is_enumerated) + + nogt_opt, gt_opt = utils.zipped_keep_blocks_if(nogt, gt, pred=lambda b: 'DagSolvedOptimally' in b) + + nogt_rel, gt_rel = blk_relative_cost(nogt, gt) + + result = { + 'Total Blocks in Benchsuite': utils.count(nogt_all), + 'Num Blocks enumerated with & without GT': utils.count(nogt), + 'Num Blocks proved optimal just by GT': NUM_PROVED_OPTIMAL_WITHOUT_ENUMERATING, + + 'Total Compile Time (s) (all benchsuite) (No GT)': total_compile_time_seconds(nogt_all), + 'Total Compile Time (s) (all benchsuite) (GT)': total_compile_time_seconds(gt_all), + + 'Total Sched Time (No GT)': sched_time(nogt), + 'Total Sched Time (GT)': sched_time(gt), + 'Enum Time (No GT)': utils.sum_stat_for_all(enum_time_for_blk, nogt), + 'Enum Time (GT)': utils.sum_stat_for_all(enum_time_for_blk, gt), + 'Lower Bound Time (No GT)': compile_times.first_lower_bound_time(nogt), + 'Lower Bound Time (GT)': compile_times.first_lower_bound_time(gt), + 'Heuristic Time (No GT)': compile_times.first_heuristic_time(nogt), + 'Heuristic Time (GT)': compile_times.first_heuristic_time(gt), + 'Total GT Time': utils.sum_stat_for_all(total_gt_elapsed_for_blk, gt), + 'Edges Added': utils.sum_stat_for_all(edges_added_for_blk, gt), + 'Edges Removed': utils.sum_stat_for_all(edges_removed_for_blk, gt), + 'Edges Rejected by RP': utils.sum_stat_for_all(edges_rp_rejected_for_blk, gt), + + 'Total Sched Time (opt. blks only) (No GT)': sched_time(nogt_opt), + 'Total Sched Time (opt. blks only) (GT)': sched_time(gt_opt), + 'Enum Time (opt. blocks only) (No GT)': utils.sum_stat_for_all(enum_time_for_blk, nogt_opt), + 'Enum Time (opt. blocks only) (GT)': utils.sum_stat_for_all(enum_time_for_blk, gt_opt), + + 'Lower Bound Time (opt. blocks only) (No GT)': compile_times.first_lower_bound_time(nogt_opt), + 'Lower Bound Time (opt. blocks only) (GT)': compile_times.first_lower_bound_time(gt_opt), + 'Heuristic Time (opt. blocks only) (No GT)': compile_times.first_heuristic_time(nogt_opt), + 'Heuristic Time (opt. blocks only) (GT)': compile_times.first_heuristic_time(gt_opt), + 'Total GT Time (opt. blocks only)': utils.sum_stat_for_all(total_gt_elapsed_for_blk, gt_opt), + 'Edges Added (opt. blocks only)': utils.sum_stat_for_all(edges_added_for_blk, gt_opt), + 'Edges Removed (opt. blocks only)': utils.sum_stat_for_all(edges_removed_for_blk, gt_opt), + 'Edges Rejected by RP (opt. blocks only)': utils.sum_stat_for_all(edges_rp_rejected_for_blk, gt_opt), + + 'Block Cost - Relative (No GT)': nogt_rel, + 'Block Cost - Relative (GT)': gt_rel, + 'Block Cost (No GT)': utils.sum_stat_for_all(block_stats.block_cost, nogt), + 'Block Cost (GT)': utils.sum_stat_for_all(block_stats.block_cost, gt), + + 'Num Nodes Examined (opt. blocks only) (No GT)': utils.sum_stat_for_all(block_stats.nodes_examined_for_blk, nogt_opt), + 'Num Nodes Examined (opt. blocks only) (GT)': utils.sum_stat_for_all(block_stats.nodes_examined_for_blk, gt_opt), + + 'Num Timeout Unimproved (No GT)': utils.count(blk for blk in nogt + if block_stats.is_timed_out(blk) + and not block_stats.is_improved(blk)), + 'Num Timeout Unimproved (GT)': utils.count(blk for blk in gt + if block_stats.is_timed_out(blk) + and not block_stats.is_improved(blk)), + 'Num Timeout Improved (No GT)': utils.count(blk for blk in nogt + if block_stats.is_timed_out(blk) + and block_stats.is_improved(blk)), + 'Num Timeout Improved (GT)': utils.count(blk for blk in gt + if block_stats.is_timed_out(blk) + and block_stats.is_improved(blk)), + } + + return result + + +if __name__ == "__main__": + import sys + import csv + + parser = argparse.ArgumentParser() + parser.add_argument('nogt') + parser.add_argument('gt') + parser.add_argument('--pass-num', type=int, default=None, help='Which pass to analyze (default: all passes)') + ioutils.add_output_format_arg(parser) + args = analyze.parse_args(parser, 'nogt', 'gt') + + results = utils.foreach_bench( + compute_stats, args.nogt, args.gt, + pass_num=args.pass_num, + total_compile_time_seconds=compile_times.total_compile_time_seconds_f(args.benchsuite), + ) + + args.format(sys.stdout, results) diff --git a/util/gt_analysis/gt_cmp_opt_only.py b/util/gt_analysis/gt_cmp_opt_only.py new file mode 100755 index 00000000..107f0bce --- /dev/null +++ b/util/gt_analysis/gt_cmp_opt_only.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import argparse + +import analyze +from analyze import Logs, utils +from analyze.lib import block_stats +from gt_analysis import gt_cmp + +is_optimal = block_stats.is_optimal + + +def compute_stats(nogt: Logs, gt: Logs): + nogt, gt = utils.zipped_keep_blocks_if(nogt, gt, pred=is_optimal) + return gt_cmp.compute_stats(nogt, gt) + + +if __name__ == "__main__": + import sys + import csv + + parser = argparse.ArgumentParser() + parser.add_argument('nogt') + parser.add_argument('gt') + args = analyze.parse_args(parser, 'nogt', 'gt') + + results = utils.foreach_bench(compute_stats, args.nogt, args.gt) + + writer = csv.DictWriter(sys.stdout, + fieldnames=['Benchmark'] + list(results['Total'].keys())) + writer.writeheader() + for bench, bench_res in results.items(): + writer.writerow({'Benchmark': bench, **bench_res}) diff --git a/util/misc/json2infolog.py b/util/misc/json2infolog.py old mode 100644 new mode 100755 diff --git a/util/misc/load_logs.py b/util/misc/load_logs.py new file mode 100644 index 00000000..81b56c22 --- /dev/null +++ b/util/misc/load_logs.py @@ -0,0 +1,10 @@ +import argparse +import analyze + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('logs', nargs='+', help='The logs to analyze') + args = analyze.parse_args(parser, 'logs') + logs = args.logs + + print('Parsed logs into variable `logs`') diff --git a/util/misc/merge-csv.py b/util/misc/merge-csv.py new file mode 100755 index 00000000..c6e9daf1 --- /dev/null +++ b/util/misc/merge-csv.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import argparse +import csv +import sys + + +def main(infile, outfile): + metrics = {} + metric_names = [] + + for metric, total, bench in csv.reader(infile): + assert total == bench or total == 'Total' + if metric not in metrics: + metric_names.append(metric) + + metrics.setdefault(metric, []).append(bench) + + writer = csv.writer(outfile) + for metric in metric_names: + try: + writer.writerow([metric, sum(int(x) for x in metrics[metric]), *metrics[metric]]) + except ValueError: + writer.writerow([metric, 'Total', *metrics[metric]]) + + +if __name__ == '__main__': + main(sys.stdin, sys.stdout) diff --git a/util/misc/raw-spill-counts.py b/util/misc/raw-spill-counts.py new file mode 100755 index 00000000..61829608 --- /dev/null +++ b/util/misc/raw-spill-counts.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import argparse +import enum +import re +import sys +from typing import Callable, IO, List +from pathlib import Path +from contextlib import contextmanager + + +@contextmanager +def argfile(filename: str, mode: str): + if filename == '-': + yield sys.stdin if mode == 'r' else sys.stdout + else: + with open(filename, mode) as f: + yield f + + +class SpillStat(enum.Enum): + RAW = re.compile(r'Function: (?P\S*?)\nGREEDY RA: Number of spilled live ranges: (?P\d+)') + WEIGHTED = re.compile(r'SC in Function (?P\S*?) (?P-?\d+)') + + +def _sum_stat(s, r: re.Match, fn_filter: Callable[[str, int], bool]) -> int: + return sum(int(m['value']) for m in r.finditer(s) if fn_filter(m['name'], int(m['value']))) + + +def sum_stat(infile: IO, r: re.Match, *, fn_filter: Callable[[str, int], bool] = lambda k, v: True) -> int: + try: + pos = infile.tell() + return _sum_stat(infile.read(), r, fn_filter) + except MemoryError: + infile.seek(pos) + return sum( + _sum_stat(line, r, fn_filter) for line in infile + ) + + +def main(infile: IO, outfile: IO, which: SpillStat = SpillStat.RAW, *, fn_filter: Callable[[str, int], bool] = lambda k, v: True): + spill_count = sum_stat(infile, which.value, fn_filter=fn_filter) + print(spill_count, file=outfile) + + +def raw_main(argv: List[str]) -> None: + parser = argparse.ArgumentParser(description='Extract spill counts') + parser.add_argument('--which', default='raw', choices=('weighted', 'raw'), + help='Whether to extract weighted or raw spills only. Default: raw') + parser.add_argument('--hot-only', help='A file with a space-separated list of functions to consider in the count') + parser.add_argument('-o', '--output', default='-', + help='Where to output the information to, - for stdout. Defaults to stdout') + parser.add_argument('file', help='The file to process, - for stdin.') + + args = parser.parse_args(argv) + + if args.hot_only: + content = Path(args.hot_only).read_text() + hot_fns = set(content.split()) + def fn_filter(k, v): return k in hot_fns + else: + def fn_filter(k, v): return True + + with argfile(args.file, 'r') as infile, \ + argfile(args.output, 'w') as outfile: + main(infile, outfile, SpillStat[args.which.upper()], fn_filter=fn_filter) + + +if __name__ == '__main__': + raw_main(sys.argv[1:]) diff --git a/util/misc/validation-test.py b/util/misc/validation-test.py index b4e15f79..c28276b9 100755 --- a/util/misc/validation-test.py +++ b/util/misc/validation-test.py @@ -1,157 +1,378 @@ -#/usr/bin/python3 -# TODO -# 1: Add options praser. -# 2: Make printing all mismatched dags optional and disabled by default. -# 3: Add option to print out x number of blocks with largest mismatches. -# 4: Add option to print out x number of mismatches with smallest number of instructions. - -import os, sys +# /usr/bin/python3 + +import sys import itertools +from typing import Callable, Dict, List, Optional, Tuple +from dataclasses import dataclass, field +from collections import defaultdict +from enum import Enum +from textwrap import dedent +import argparse + +import analyze +from analyze import Logs, Block + + +class BlockProcessingError(Exception): + block: Block + + def __init__(self, message: str, block: Block): + self.block = block + super().__init__(f'{message}:\n{block.raw_log}') + + +@dataclass +class DagInfo: + id: str + benchmark: str + num_instructions: int + pass_num: int + + lower_bound: int + relative_cost: int + length: int + is_optimal: bool + + @property + def cost(self): + return self.lower_bound + self.relative_cost + + +class MismatchKind(Enum): + BOTH_OPTIMAL_BUT_UNEQUAL = 0 + FIRST_OPTIMAL_BUT_WORSE = 1 + SECOND_OPTIMAL_BUT_WORSE = 2 + + +@dataclass +class Mismatch: + # The dag id: function name + basic block number + dag_id: str + # Which benchmark this region comes from + benchmark: str + # The number of instructions for this region + region_size: int + + # The cost information indexed by "first" == 0, "second" == 1 + lengths: Tuple[int, int] + costs: Tuple[int, int] + + kind: MismatchKind + + +@dataclass +class ValidationInfo: + num_regions_first: int = 0 + num_regions_second: int = 0 + + num_optimal_both: int = 0 + num_optimal_first: int = 0 + num_optimal_second: int = 0 + + num_mismatch: Dict[MismatchKind, int] = field(default_factory=lambda: defaultdict(lambda: 0)) + + mismatches: List[Mismatch] = field(default_factory=list) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from readlogs import * # Explain this many of the blocks missing a lower bound MISSING_LOWER_BOUND_DUMP_COUNT = 3 MISSING_LOWER_BOUND_DUMP_LINES = 10 -dags1 = {} -dags2 = {} +# If there is no PassFinished, what pass "number" should we consider this to be? +DEFAULT_PASS = [{'num': 0}] + + +def split_adjacent(iterable, adj_eq=None): + ''' + Splits the iterable into regions of "equal values" as specified by adj_eq. + + Examples: + split_adjacent([1, 1, 1, 2, 2, 2, 2, 2, 3, 5]) # -> [(1, 1, 1), (2, 2, 2, 2, 2), (3,), (5,)] + split_adjacent([1, 2, 3, 4, 1, 2, 3, 4, 2, 3, 8, 9], lambda x, y: x <= y) + # -> [(1, 2, 3, 4), (1, 2, 3, 4), (2, 3, 8, 9)] + ''' + if adj_eq is None: + # Default: use == + def adj_eq(x, y): return x == y + + values = [] + for x in iterable: + if values and not adj_eq(values[-1], x): + yield tuple(values) + values.clear() + values.append(x) + + assert len(values) > 0 + yield tuple(values) + + +def pass_num(block) -> int: + return block.get('PassFinished', DEFAULT_PASS)[0]['num'] + -def dags_info(logtext): +def try_first(block: Block, *event_ids): + for index, event_id in enumerate(event_ids): + try: + return block[event_id] + except KeyError: + if index == len(event_ids) - 1: + raise + + +def extract_dag_info(logs: Logs) -> Dict[str, List[List[DagInfo]]]: dags = {} - unfiltered = [keep_only_singular_events(block) for block in parse_blocks(logtext)] - blocks = [block for block in unfiltered if 'CostLowerBound' in block] + blocks = list(logs) + + no_lb = [block for block in blocks if 'CostLowerBound' not in block] - if len(blocks) != len(unfiltered): + if no_lb: print('WARNING: Missing a logged lower bound for {missing}/{total} blocks.' - .format(missing=len(unfiltered) - len(blocks), total=len(unfiltered)), file=sys.stderr) + .format(missing=len(no_lb), total=len(blocks)), file=sys.stderr) - missing = set(unfiltered) - set(blocks) - trimmed = ('\n'.join(block.splitlines()[:MISSING_LOWER_BOUND_DUMP_LINES]) for block in missing) + trimmed = ('\n'.join(block.raw_log.splitlines()[:MISSING_LOWER_BOUND_DUMP_LINES]) for block in no_lb) for i, block in enumerate(itertools.islice(trimmed, MISSING_LOWER_BOUND_DUMP_COUNT)): print('WARNING: block {} missing lower-bound:\n{}\n...'.format(i, block), file=sys.stderr) for block in blocks: - lowerBound = block['CostLowerBound']['cost'] - blockInfo = block['BestResult'] - dagName = blockInfo['name'] - dags[dagName] = { - 'lowerBound': lowerBound, - 'cost': blockInfo['cost'] + lowerBound, - 'relativeCost': blockInfo['cost'], - 'length': blockInfo['length'], - 'isOptimal': blockInfo['optimal'] - } + try: + best_result = try_first(block, 'BestResult', 'HeuristicResult')[-1] + is_optimal = best_result.get('optimal', False) or best_result['cost'] == 0 or \ + 'INFO: Marking SLIL list schedule as optimal due to zero PERP.' in block.raw_log + + dags.setdefault(block.name, []).append(DagInfo( + id=block.name, + benchmark=block.benchmark, + num_instructions=block.single('ProcessDag')['num_instructions'], + pass_num=pass_num(block), + lower_bound=block['CostLowerBound'][-1]['cost'], + relative_cost=best_result['cost'], + length=best_result['length'], + is_optimal=is_optimal, + )) + except Exception as ex: + raise BlockProcessingError('Failed when processing block', block) from ex + + for k, block_passes in dags.items(): + # Safe to modify dags while iterating because we use .items() to get a copy + dags[k] = list(map(list, split_adjacent(block_passes, lambda x, y: x.pass_num < y.pass_num))) return dags -with open(str(sys.argv[1])) as logfile1: - dags1 = dags_info(logfile1.read()) - -with open(str(sys.argv[2])) as logfile2: - dags2 = dags_info(logfile2.read()) - -numDagsLog1 = len(dags1) -numDagsLog2 = len(dags2) -# The number of blocks that are optimal in both logs. -optimalInBoth = 0 -# The number of blocks that are only optimal in log 1. -optimalLog1 = 0 -# The number of blocks that are only optimal in log 2. -optimalLog2 = 0 -# Mismatches where blocks are optimal in both logs but have different costs. -misNonEqual = 0 -# Mismatches where block is optimal in log 1 but it has a higher cost than the non-optimal block in log 2. -misBlk1Opt = 0 -# Mismatches where block is optimal in log 2 but it has a higher cost than the non-optimal block in log 1. -misBlk2Opt = 0 +def parse_mismatch(blk1: DagInfo, blk2: DagInfo) -> Optional[Mismatch]: + mismatch = Mismatch( + dag_id=blk1.id, + benchmark=blk1.benchmark, + region_size=blk1.num_instructions, + lengths=(blk1.length, blk2.length), + costs=(blk1.cost, blk2.cost), + kind=None, + ) + if blk1.is_optimal and blk2.is_optimal: + mismatch.kind = MismatchKind.BOTH_OPTIMAL_BUT_UNEQUAL + return mismatch if blk1.cost != blk2.cost else None + elif blk1.is_optimal: + mismatch.kind = MismatchKind.FIRST_OPTIMAL_BUT_WORSE + return mismatch if blk1.cost > blk2.cost else None + elif blk2.is_optimal: + mismatch.kind = MismatchKind.SECOND_OPTIMAL_BUT_WORSE + return mismatch if blk2.cost > blk1.cost else None + else: + return None + + +def classify_optimal(out: ValidationInfo, blk1: DagInfo, blk2: DagInfo): + if blk1.is_optimal and blk2.is_optimal: + out.num_optimal_both += 1 + elif blk1.is_optimal: + out.num_optimal_first += 1 + elif blk2.is_optimal: + out.num_optimal_second += 1 + + +def classify_mismatch(out: ValidationInfo, mismatch: Mismatch): + out.num_mismatch[mismatch.kind] += 1 + + +def validate_dags(dags1: Dict[str, List[List[DagInfo]]], dags2: Dict[str, List[List[DagInfo]]]) -> ValidationInfo: + result = ValidationInfo(num_regions_first=len(dags1), num_regions_second=len(dags2)) + + for region_f, region_s in zip(dags1.items(), dags2.items()): + name_f, grouped_blocks_f = region_f + name_s, grouped_blocks_s = region_s + + for blocks_f, blocks_s in zip(grouped_blocks_f, grouped_blocks_s): + # blocks_* is the groups of blocks referring to the same problem, with different pass nums. + blocks = list(zip(blocks_f, blocks_s)) + + block_f, block_s = blocks[0] + classify_optimal(result, block_f, block_s) + + mismatch = parse_mismatch(block_f, block_s) + if mismatch is not None: + classify_mismatch(result, mismatch) + result.mismatches.append(mismatch) + + for next_block_f, next_block_s in blocks[1:]: + if not block_f.is_optimal: + next_block_f.is_optimal = False + if not block_s.is_optimal: + next_block_s.is_optimal = False + + classify_optimal(result, next_block_f, next_block_s) + mismatch = parse_mismatch(next_block_f, next_block_s) + if mismatch is not None: + classify_mismatch(result, mismatch) + result.mismatches.append(mismatch) + + block_f, block_s = next_block_f, next_block_s + + return result + + +def print_mismatches(info: ValidationInfo, + print_stats_info: Callable[[ValidationInfo], None], + print_mismatch_info: Callable[[ValidationInfo], None], + print_mismatch_summaries: List[Callable[[List[Mismatch]], None]]): + print_stats_info(info) + print_mismatch_info(info) + + if info.mismatches: + for print_summary in print_mismatch_summaries: + print_summary(info.mismatches) + + +def enable_if(cond: bool): + def wrapped(f): + return f if cond else lambda *args: None + + return wrapped + + # The quantity of blocks with the largest mismatches to print. -numLarMisPrt = 10 +NUM_LARGEST_MISMATCHES_PRINT = 10 # The quantity of mismatched blocks with the shortest length to print. -numSmlBlkPrt = 50 -# Dictionary with the sizes of the mismatches for each mismatched block and the size of the block. -mismatches = {} - - - -if numDagsLog1 != numDagsLog2: - print('Error: Different number of dags in each log file.') - -for dagName in dags1: - if dagName not in dags2: - print('Error: Could not find ' + dagName + ' in the second log file.') - continue - - dag1 = dags1[dagName] - dag2 = dags2[dagName] - if dag1['isOptimal'] and dag2['isOptimal']: - optimalInBoth+=1 - if dag1['cost'] != dag2['cost']: - # There was a mismatch where blocks are optimal in both logs but have different costs - misNonEqual += 1 - mismatches[dagName] = {} - mismatches[dagName]['length'] = dag1['length'] - mismatches[dagName]['misSize'] = abs(dag1['cost'] - dag2['cost']) - #print('Mismatch for dag ' + dagName + ' (Both optimal with non-equal cost)') - - elif dag1['isOptimal']: - optimalLog1+=1 - if dag1['cost'] > dag2['cost']: - # There was a mismatch where block is optimal in log 1 but it has a higher cost than the non-optimal block in log 2 - misBlk1Opt += 1 - mismatches[dagName] = {} - mismatches[dagName]['length'] = dag1['length'] - mismatches[dagName]['misSize'] = dag1['cost'] - dag2['cost'] - #print('Mismatch for dag ' + dagName + ' (Only optimal in log 1 but has higher cost than the non-optimal block in log 2)') - - elif dag2['isOptimal']: - optimalLog2+=1 - if dag2['cost'] > dag1['cost']: - # There was a mismatch where block is optimal in log 2 but it has a higher cost than the non-optimal block in log 1 - misBlk2Opt += 1 - mismatches[dagName] = {} - mismatches[dagName]['length'] = dag1['length'] - mismatches[dagName]['misSize'] = dag2['cost'] - dag1['cost'] - #print('Mismatch for dag ' + dagName + ' (Only optimal in log 2 but has higher cost than the non-optimal block in log 1)') - -print('Optimal Block Stats') -print('-----------------------------------------------------------') -print('Blocks in log file 1: ' + str(numDagsLog1)) -print('Blocks in log file 2: ' + str(numDagsLog2)) -print('Blocks that are optimal in both files: ' + str(optimalInBoth)) -print('Blocks that are optimal in log 1 but not in log 2: ' + str(optimalLog1)) -print('Blocks that are optimal in log 2 but not in log 1: ' + str(optimalLog2)) -print('----------------------------------------------------------\n') - -print('Mismatch stats') -print('-----------------------------------------------------------') -print('Mismatches where blocks are optimal in both logs but have different costs: ' + str(misNonEqual)) -print('Mismatches where the block is optimal in log 1 but it has a higher cost than the non-optimal block in log 2: ' + str(misBlk1Opt)) -print('Mismatches where the block is optimal in log 2 but it has a higher cost than the non-optimal block in log 1: ' + str(misBlk2Opt)) -print('Total mismatches: ' + str(misNonEqual + misBlk1Opt + misBlk2Opt)) -print('-----------------------------------------------------------\n') - -print('The ' + str(numLarMisPrt) + ' mismatched blocks with the largest difference in cost') -print('-----------------------------------------------------------') -sortedMaxMis = sorted(mismatches.items(), key=lambda i: (mismatches[i[0]]['misSize'], i[0]), reverse=True) -i = 1 -for block in sortedMaxMis[:numLarMisPrt]: - print(str(i) + ':') - print('Block Name: ' + block[0] + '\nLength: ' + str(block[1]['length']) + '\nDifference in cost: ' + str(block[1]['misSize'])) - i += 1 -print('-----------------------------------------------------------\n') - -print('The smallest ' + str(numSmlBlkPrt) + ' mismatched blocks') -print('-----------------------------------------------------------') -sortedMisSize = sorted(mismatches.items(), key=lambda i: (mismatches[i[0]]['length'], i[0])) -i = 1 -for block in sortedMisSize[:numSmlBlkPrt]: - print(str(i) + ':') - print('Block Name: ' + block[0] + '\nLength: ' + str(block[1]['length']) + '\nDifference in cost: ' + str(block[1]['misSize'])) - i += 1 -print('-----------------------------------------------------------') +NUM_SMALLEST_BLOCKS_PRINT = 50 + + +def main(first, second, + quiet: bool = False, + summarize_biggest_cost_difference: bool = True, + summarize_smallest_regions: bool = True): + dags1 = extract_dag_info(first) + dags2 = extract_dag_info(second) + info: ValidationInfo = validate_dags(dags1, dags2) + + @enable_if(not quiet) + def print_stats_info(info: ValidationInfo): + print('Optimal Block Stats') + print('-----------------------------------------------------------') + print('Blocks in log file 1: ' + str(info.num_regions_first)) + print('Blocks in log file 2: ' + str(info.num_regions_second)) + print('Blocks that are optimal in both files: ' + str(info.num_optimal_both)) + print('Blocks that are optimal in log 1 but not in log 2: ' + str(info.num_optimal_first)) + print('Blocks that are optimal in log 2 but not in log 1: ' + str(info.num_optimal_second)) + print('----------------------------------------------------------\n') + + @enable_if(info.mismatches or not quiet) + def print_mismatch_info(info: ValidationInfo): + print('Mismatch stats') + print('-----------------------------------------------------------') + print('Mismatches where blocks are optimal in both logs but have different costs: ' + + str(info.num_mismatch[MismatchKind.BOTH_OPTIMAL_BUT_UNEQUAL])) + print('Mismatches where the block is optimal in log 1 but it has a higher cost than the non-optimal block in log 2: ' + + str(info.num_mismatch[MismatchKind.FIRST_OPTIMAL_BUT_WORSE])) + print('Mismatches where the block is optimal in log 2 but it has a higher cost than the non-optimal block in log 1: ' + + str(info.num_mismatch[MismatchKind.SECOND_OPTIMAL_BUT_WORSE])) + print('Total mismatches: ' + str(len(info.mismatches))) + print('-----------------------------------------------------------\n') + + def print_block_info(index: int, mismatch: Mismatch): + cost_diff = mismatch.costs[0] - mismatch.costs[1] + print(dedent(f'''\ + {index}: + Block Name: {mismatch.dag_id} + Benchmark: {mismatch.benchmark} + Num Instructions: {mismatch.region_size} + Length: {mismatch.lengths[0]} --> {mismatch.lengths[1]} + Difference in cost: {cost_diff} + Percent cost difference: {(cost_diff / mismatch.costs[0])*100:0.2f} % + ''' + )) + + @enable_if(summarize_biggest_cost_difference) + def print_big_diff_summary(mismatches: List[Mismatch]): + if NUM_LARGEST_MISMATCHES_PRINT == 0: + print('Requested 0 mismatched blocks with the largest difference in cost') + return + + print('The ' + str(NUM_LARGEST_MISMATCHES_PRINT) + ' mismatched blocks with the largest difference in cost') + print('-----------------------------------------------------------') + sortedMaxMis = sorted(mismatches, key=lambda m: abs(m.costs[1] - m.costs[0]), reverse=True) + for index, mismatch in enumerate(sortedMaxMis[:NUM_LARGEST_MISMATCHES_PRINT]): + print_block_info(index, mismatch) + print('-----------------------------------------------------------\n') + + @enable_if(summarize_smallest_regions) + def print_small_summary(mismatches: List[Mismatch]): + if NUM_SMALLEST_BLOCKS_PRINT == 0: + print('Requested 0 mismatched blocks with the smallest block size') + return + + print('The smallest ' + str(NUM_SMALLEST_BLOCKS_PRINT) + ' mismatched blocks') + print('-----------------------------------------------------------') + sortedMisSize = sorted(mismatches, key=lambda m: m.region_size) + for index, mismatch in enumerate(sortedMisSize[:NUM_LARGEST_MISMATCHES_PRINT]): + print_block_info(index, mismatch) + print('-----------------------------------------------------------\n') + + print_mismatches( + info, + print_stats_info=print_stats_info, + print_mismatch_info=print_mismatch_info, + print_mismatch_summaries=[print_big_diff_summary, print_small_summary] + ) + if info.mismatches: + exit(f'{len(info.mismatches)} mismatches found') + + +if __name__ == "__main__": + dags1 = {} + dags2 = {} + + parser = argparse.ArgumentParser() + parser.add_argument('first') + parser.add_argument('second') + + parser.add_argument('-q', '--quiet', action='store_true', + help='Only print mismatch info, and only if there are mismatches') + parser.add_argument('--no-summarize-largest-cost-difference', action='store_true', + help='Do not summarize the mismatches with the biggest difference in cost') + parser.add_argument('--no-summarize-smallest-mismatches', action='store_true', + help='Do not summarize the mismatches with the smallest region size') + + parser.add_argument('--num-largest-cost-mismatches-print', type=int, default=10, + help='The number of mismatches blocks with the largest (by cost) mismatches to print') + parser.add_argument('--num-smallest-mismatches-print', type=int, default=10, + help='The number of mismatched blocks with the shortest length to print') + + parser.add_argument('--missing-lb-dump-count', type=int, default=3, + help='The number of blocks with missing lower bounds to display') + parser.add_argument('--missing-lb-dump-lines', type=int, default=10, + help='The number of lines of a block with missing lower bound to display') + args = analyze.parse_args(parser, 'first', 'second') + + NUM_LARGEST_MISMATCHES_PRINT = args.num_largest_cost_mismatches_print + NUM_SMALLEST_BLOCKS_PRINT = args.num_smallest_mismatches_print + MISSING_LOWER_BOUND_DUMP_COUNT = args.missing_lb_dump_count + MISSING_LOWER_BOUND_DUMP_LINES = args.missing_lb_dump_lines + + main( + args.first, args.second, + quiet=args.quiet, + summarize_biggest_cost_difference=not args.no_summarize_largest_cost_difference, + summarize_smallest_regions=not args.no_summarize_smallest_mismatches, + )