Support reporting geometric mean by benchmark tags

Addresses python/pyperformance#208

This reports geometric mean organized by the tag(s) assigned to each benchmark.
This will allow us to include benchmarks in the pyperformance suite that we
don't necessarily want to include in "one big overall number" to represent progress.

pyperf/_compare.py

@@ -49,6 +49,31 @@ def format_geometric_mean(norm_means):
     return format_normalized_mean(geo_mean)
 
 
+class TaggedGroups(object):
+    """
+    A class to group benchmark results based on their tags to aid in the
+    computation of geometric means. A special group 'all' includes all
+    benchmarks.
+    """
+    def __init__(self, all_results):
+        self.tags = set()
+        for results in all_results:
+            for result in results:
+                self.tags.update(result.ref.benchmark.get_metadata().get("tags", []))
+        self.tags = sorted(list(self.tags))
+
+        self.collections = {}
+        for tag in ["all"] + self.tags:
+            self.collections[tag] = [[] for x in range(len(all_results[0]))]
+
+    def add_result(self, index, result):
+        for tag in ["all"] + result.ref.benchmark.get_metadata().get("tags", []):
+            self.collections[tag][index].append(result.norm_mean)
+
+    def get_results(self):
+        return self.collections.items()
+
+
 class CompareResult(object):
     def __init__(self, ref, changed, min_speed=None):
         # CompareData object
@@ -274,9 +299,7 @@ def sort_key(results):
         for item in self.all_results[0]:
             headers.append(item.changed.name)
 
-        all_norm_means = []
-        for column in headers[2:]:
-            all_norm_means.append([])
+        tagged_groups = TaggedGroups(self.all_results)
 
         rows = []
         not_significant = []
@@ -298,7 +321,7 @@ def sort_key(results):
                 else:
                     text = "not significant"
                 significants.append(significant)
-                all_norm_means[index].append(result.norm_mean)
+                tagged_groups.add_result(index, result)
                 row.append(text)
 
             if any(significants):
@@ -308,13 +331,14 @@ def sort_key(results):
 
         # only compute the geometric mean if there is at least two benchmarks
         # and if at least one is signicant.
-        if len(all_norm_means[0]) > 1 and rows:
-            row = ['Geometric mean', '(ref)']
-            for norm_means in all_norm_means:
-                row.append(format_geometric_mean(norm_means))
-            rows.append(row)
-
         if rows:
+            for tag, norm_mean in tagged_groups.get_results():
+                if len(norm_mean[0]) > 1:
+                    row = ['Geometric mean (%s)' % tag, '(ref)']
+                    for values in norm_mean:
+                        row.append(format_geometric_mean(values))
+                    rows.append(row)
+
             if self.table_format == 'rest':
                 table = ReSTTable(headers, rows)
             else:
@@ -419,29 +443,29 @@ def list_ignored(self):
     def compare_geometric_mean(self):
         all_results = self.all_results
 
-        # use a list since two filenames can be identical,
-        # even if results are different
-        all_norm_means = []
-        for item in all_results[0]:
-            all_norm_means.append((item.changed.name, []))
+        if len(all_results) < 2:
+            # only compute the geometric mean when there is at least two benchmarks
+            return
+
+        tagged_groups = TaggedGroups(all_results)
 
         for results in all_results:
             for index, result in enumerate(results):
-                all_norm_means[index][1].append(result.norm_mean)
-
-        if len(all_norm_means[0][1]) < 2:
-            # only compute the geometric mean when there is at least two benchmarks
-            return
+                tagged_groups.add_result(index, result)
 
         print()
-        if len(all_norm_means) > 1:
+        if len(all_results[0]) > 1:
             display_title('Geometric mean')
-            for name, norm_means in all_norm_means:
-                geo_mean = format_geometric_mean(norm_means)
-                print(f'{name}: {geo_mean}')
+            for tag, norm_means in tagged_groups.get_results():
+                print(f"'{tag}':")
+                for item, norm_mean in zip(all_results[0], norm_means):
+                    name = item.changed.name
+                    geo_mean = format_geometric_mean(norm_mean)
+                    print(f'  {name}: {geo_mean}')
         else:
-            geo_mean = format_geometric_mean(all_norm_means[0][1])
-            print(f'Geometric mean: {geo_mean}')
+            for tag, norm_means in tagged_groups.get_results():
+                geo_mean = format_geometric_mean(norm_means[0])
+                print(f"Geometric mean '{tag}': {geo_mean}")
 
     def compare(self):
         if self.table:

pyperf/_metadata.py

@@ -42,6 +42,12 @@ def is_positive(value):
     return (value >= 0)
 
 
+def is_tags(value):
+    if not isinstance(value, (list, tuple)):
+        return False
+    return all(isinstance(x, str) and x != 'all' for x in value)
+
+
 def parse_load_avg(value):
     if isinstance(value, NUMBER_TYPES):
         return value
@@ -62,6 +68,7 @@ def format_noop(value):
 LOOPS = _MetadataInfo(format_number, (int,), is_strictly_positive, 'integer')
 WARMUPS = _MetadataInfo(format_number, (int,), is_positive, 'integer')
 SECONDS = _MetadataInfo(format_seconds, NUMBER_TYPES, is_positive, 'second')
+TAGS = _MetadataInfo(format_generic, (tuple, list), is_tags, 'tag')
 
 # Registry of metadata keys
 METADATA = {
@@ -84,6 +91,7 @@ def format_noop(value):
     'recalibrate_loops': LOOPS,
     'calibrate_warmups': WARMUPS,
     'recalibrate_warmups': WARMUPS,
+    'tags': TAGS,
 }
 
 DEFAULT_METADATA_INFO = _MetadataInfo(format_generic, METADATA_VALUE_TYPES, None, None)