From 5833b2382bb6dfb3f830d8b9a2c5bb529792aeb9 Mon Sep 17 00:00:00 2001
From: Tobias Udtke <tobias.udtke@gmail.com>
Date: Mon, 22 Mar 2021 15:07:09 +0100
Subject: [PATCH 1/3] test: add more comprehensive runtime tests.

---
 tests/test_schulze.py | 44 +++++++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/tests/test_schulze.py b/tests/test_schulze.py
index 3adf4ed..e8874e5 100644
--- a/tests/test_schulze.py
+++ b/tests/test_schulze.py
@@ -1,6 +1,6 @@
 import datetime
 import random
-from typing import Dict, List, Optional, Tuple
+from typing import Collection, Dict, List, Optional, Tuple
 import unittest
 
 from schulze_condorcet import schulze_evaluate
@@ -95,25 +95,29 @@ def test_schulze_runtime(self) -> None:
         # silly test, since I just realized, that the algorithm runtime is
         # linear in the number of votes, but a bit more scary in the number
         # of candidates
-        candidates = ('0', '1', '2', '3', '4')
-        votes = []
-        for _ in range(2000):
-            parts = list(candidates)
-            random.shuffle(parts)
-            relations = (random.choice(('=', '>'))
-                         for _ in range(len(candidates)))
-            vote = ''.join(c + r for c, r in zip(candidates, relations))
-            votes.append(vote[:-1])
-        times = {}
-        for num in (10, 100, 1000, 2000):
-            start = datetime.datetime.utcnow()
-            for _ in range(10):
-                schulze_evaluate(votes[:num], candidates)
-            stop = datetime.datetime.utcnow()
-            times[num] = stop - start
-        reference = datetime.timedelta(milliseconds=5)
-        for num, delta in times.items():
-            self.assertGreater(num * reference, delta)
+        num_evaluation_runs = 3
+        reference = datetime.timedelta(microseconds=2)
+
+        def create_random_votes(candidates: Collection[str], n: int) -> List[str]:
+            return [''.join(c + random.choice(("=", ">")) for c in random.sample(candidates, len(candidates)))[:-1]
+                    for _ in range(n)]
+
+        for num_candidates in (3, 5, 10, 20):
+            candidates = tuple(map(str, range(num_candidates)))
+            for num_votes in (100, 1000, 2000):
+                votes = create_random_votes(candidates, num_votes)
+                # Evaluation time depends linearly on number of votes and quadratically on number of candidates.
+                time_limit = num_votes * num_candidates ** 2 * reference * num_evaluation_runs
+                for metric in (margin, winning_votes):
+                    with self.subTest(c=num_candidates, v=num_votes, m=metric.__name__):
+                        runtimes = []
+                        for _ in range(num_evaluation_runs):
+                            start = datetime.datetime.utcnow()
+                            schulze_evaluate(votes, candidates, metric)
+                            runtimes.append(datetime.datetime.utcnow() - start)
+                        total_runtime = sum(runtimes, datetime.timedelta())
+                        self.assertLess(total_runtime, time_limit)
+                        self.assertGreater(10 * total_runtime, time_limit)
 
 
 if __name__ == '__main__':

From f469f302a70dfd54003e0c9dad2118a7a07c3803 Mon Sep 17 00:00:00 2001
From: Tobias Udtke <tobias.udtke@gmail.com>
Date: Mon, 22 Mar 2021 15:12:33 +0100
Subject: [PATCH 2/3] tests: make mypy happy

---
 tests/test_schulze.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_schulze.py b/tests/test_schulze.py
index e8874e5..3e2bc94 100644
--- a/tests/test_schulze.py
+++ b/tests/test_schulze.py
@@ -1,6 +1,6 @@
 import datetime
 import random
-from typing import Collection, Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Sequence, Tuple
 import unittest
 
 from schulze_condorcet import schulze_evaluate
@@ -98,7 +98,7 @@ def test_schulze_runtime(self) -> None:
         num_evaluation_runs = 3
         reference = datetime.timedelta(microseconds=2)
 
-        def create_random_votes(candidates: Collection[str], n: int) -> List[str]:
+        def create_random_votes(candidates: Sequence[str], n: int) -> List[str]:
             return [''.join(c + random.choice(("=", ">")) for c in random.sample(candidates, len(candidates)))[:-1]
                     for _ in range(n)]
 

From f2d4991d0fe046a47f674eb9f3da95cb5c5cb6f9 Mon Sep 17 00:00:00 2001
From: Tobias Udtke <tobias.udtke@gmail.com>
Date: Mon, 22 Mar 2021 15:14:53 +0100
Subject: [PATCH 3/3] drone: separate mypy and tests steps.

---
 .drone.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index 54888a2..a4e143d 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -4,11 +4,14 @@ type: docker
 name: python-minimal
 
 steps:
-- name: test
+- name: mypy
   image: python:3.8
   commands:
   - pip install --no-cache-dir -r requirements-dev.txt
   - python3 -m mypy schulze_condorcet/ tests/
+- name: test
+  image: python:3.8
+  commands:
   - python3 -m unittest -v tests/*.py
 
 ---
@@ -17,9 +20,12 @@ type: docker
 name: python-recent
 
 steps:
-- name: test
+- name: mypy
   image: python
   commands:
   - pip install --no-cache-dir -r requirements-dev.txt
   - python3 -m mypy schulze_condorcet/ tests/
+- name: test
+  image: python
+  commands:
   - python3 -m unittest -v tests/*.py