Adjust Log Output and Elim Tree Generation (#84)

* Dump parseable log.json, update Elim tree generator to match

* Rename 'length_p_history' to 'p_history_length'

shangrla/Examples/Data/SFDA2019/SFDA2019_CandidateManifest.json

@@ -0,0 +1,40 @@
+{
+    "Version": "5.2.18.2",
+    "List": [
+        {
+            "Description": "SUZY LOFTUS",
+            "Id": 15,
+            "ExternalId": null,
+            "ContestId": 339,
+            "Type": "Regular"
+        },
+        {
+            "Description": "LEIF DAUTCH",
+            "Id": 16,
+            "ExternalId": null,
+            "ContestId": 339,
+            "Type": "Regular"
+        },
+        {
+            "Description": "NANCY TUNG",
+            "Id": 17,
+            "ExternalId": null,
+            "ContestId": 339,
+            "Type": "Regular"
+        },
+        {
+            "Description": "CHESA BOUDIN",
+            "Id": 18,
+            "ExternalId": null,
+            "ContestId": 339,
+            "Type": "Regular"
+        },
+        {
+            "Description": "Write-in",
+            "Id": 45,
+            "ExternalId": "",
+            "ContestId": 339,
+            "Type": "WriteIn"
+        }
+    ]
+}

shangrla/Examples/Data/log.json

@@ -1 +1 @@
-{"Audit": "{'seed': 12345678901234567890, 'sim_seed': 314159265, 'cvr_file': './Data/SFDA2019/SFDA2019_PrelimReport12VBMJustDASheets.raire', 'manifest_file': './Data/SFDA2019/N19 ballot manifest with WH location for RLA Upload VBM 11-14.xlsx', 'sample_file': './Data/sample.csv', 'mvr_file': './Data/mvr.json', 'log_file': './Data/log.json', 'quantile': 0.8, 'error_rate_1': 0.001, 'error_rate_2': 0.0, 'reps': 100, 'max_cards': 293555, 'strata': {'stratum_1': <shangrla.Audit.Stratum object at 0x7fc4183bf490>}}", "contests": {"339": "{'id': '339', 'name': 'DA', 'risk_limit': 0.05, 'cards': 146662, 'choice_function': 'IRV', 'n_winners': 1, 'share_to_win': None, 'candidates': ['15', '16', '17', '18', '45'], 'winner': ['15'], 'assertion_file': './Data/SFDA2019/SF2019Nov8Assertions.json', 'audit_type': 'CARD_COMPARISON', 'test': <function NonnegMean.alpha_mart at 0x7fc4190c1900>, 'g': 0.1, 'estim': <function NonnegMean.optimal_comparison at 0x7fc4190c1b40>, 'bet': None, 'use_style': True, 'assertions': {'18 v 17 elim 15 16 45': <shangrla.Audit.Assertion object at 0x7fc4190dfee0>, '17 v 16 elim 15 18 45': <shangrla.Audit.Assertion object at 0x7fc4190deec0>, '15 v 18 elim 16 17 45': <shangrla.Audit.Assertion object at 0x7fc4190de980>, '18 v 16 elim 15 17 45': <shangrla.Audit.Assertion object at 0x7fc4190dd600>, '17 v 16 elim 15 45': <shangrla.Audit.Assertion object at 0x7fc4190df7c0>, '15 v 17 elim 16 45': <shangrla.Audit.Assertion object at 0x7fc4190ded10>, '15 v 17 elim 16 18 45': <shangrla.Audit.Assertion object at 0x7fc4190def80>, '18 v 16 elim 15 45': <shangrla.Audit.Assertion object at 0x7fc4190dd720>, '15 v 16 elim 17 45': <shangrla.Audit.Assertion object at 0x7fc4190df2e0>, '15 v 16 elim 17 18 45': <shangrla.Audit.Assertion object at 0x7fc4190df4f0>, '15 v 16 elim 18 45': <shangrla.Audit.Assertion object at 0x7fc4190df610>, '15 v 16 elim 45': <shangrla.Audit.Assertion object at 0x7fc4190dfa30>, '15 v 45': <shangrla.Audit.Assertion object at 0x7fc4190dfe20>}, 'tally': None, 'sample_size': 372, 'sample_threshold': 287703018677674409350709928129415892402446167067397529607245781524786181910, 'assertion_json': [{'winner': '18', 'loser': '17', 'already_eliminated': ['15', '16', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 18 17]'}, {'winner': '17', 'loser': '16', 'already_eliminated': ['15', '18', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 17 16]'}, {'winner': '15', 'loser': '18', 'already_eliminated': ['16', '17', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 18]'}, {'winner': '18', 'loser': '16', 'already_eliminated': ['15', '17', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 18 16]'}, {'winner': '17', 'loser': '16', 'already_eliminated': ['15', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 17 16 18]'}, {'winner': '15', 'loser': '17', 'already_eliminated': ['16', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 17 18]'}, {'winner': '15', 'loser': '17', 'already_eliminated': ['16', '18', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 17]'}, {'winner': '18', 'loser': '16', 'already_eliminated': ['15', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 18 16 17]'}, {'winner': '15', 'loser': '16', 'already_eliminated': ['17', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 16 18]'}, {'winner': '15', 'loser': '16', 'already_eliminated': ['17', '18', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 16]'}, {'winner': '15', 'loser': '16', 'already_eliminated': ['18', '45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 16 17]'}, {'winner': '15', 'loser': '16', 'already_eliminated': ['45'], 'assertion_type': 'IRV_ELIMINATION', 'explanation': 'Rules out outcomes with tail [... 15 16 17 18]'}, {'winner': '15', 'loser': '45', 'already_eliminated': '', 'assertion_type': 'WINNER_ONLY', 'explanation': 'Rules out case where 15 is eliminated before 45'}], 'cvrs': 146662, 'margins': {'18 v 17 elim 15 16 45': 0.045792366120740224, '17 v 16 elim 15 18 45': 0.019902906001554532, '15 v 18 elim 16 17 45': 0.028923647570604505, '18 v 16 elim 15 17 45': 0.0830003681935334, '17 v 16 elim 15 45': 0.058079120699294995, '15 v 17 elim 16 45': 0.08064120222007065, '15 v 17 elim 16 18 45': 0.10951712099930444, '18 v 16 elim 15 45': 0.14875018750596603, '15 v 16 elim 17 45': 0.13548158350492967, '15 v 16 elim 17 18 45': 0.1365247985163165, '15 v 16 elim 18 45': 0.16666893946625572, '15 v 16 elim 45': 0.15626406294745743, '15 v 45': 0.2956457705472446}, 'p_values': {'18 v 17 elim 15 16 45': 0.47847909464463045, '17 v 16 elim 15 18 45': 0.48779308744628547, '15 v 18 elim 16 17 45': 0.44631352397209006, '18 v 16 elim 15 17 45': 2.877844184074244e-07, '17 v 16 elim 15 45': 0.0035354717267599813, '15 v 17 elim 16 45': 4.0458461637126434e-07, '15 v 17 elim 16 18 45': 2.3358046805505937e-07, '18 v 16 elim 15 45': 5.790407277111407e-13, '15 v 16 elim 17 45': 9.312863307985585e-12, '15 v 16 elim 17 18 45': 1.8495628594508403e-09, '15 v 16 elim 18 45': 1.1604860427653792e-14, '15 v 16 elim 45': 9.55405920633293e-14, '15 v 45': 1.373326785425354e-26}, 'proved': {'18 v 17 elim 15 16 45': False, '17 v 16 elim 15 18 45': False, '15 v 18 elim 16 17 45': False, '18 v 16 elim 15 17 45': True, '17 v 16 elim 15 45': True, '15 v 17 elim 16 45': True, '15 v 17 elim 16 18 45': True, '18 v 16 elim 15 45': True, '15 v 16 elim 17 45': True, '15 v 16 elim 17 18 45': True, '15 v 16 elim 18 45': True, '15 v 16 elim 45': True, '15 v 45': True}, 'max_p': 0.48779308744628547}"}}
+{"Audit": {"seed": 12345678901234567890, "sim_seed": 314159265, "cvr_file": "/Path/To/raire.txt", "manifest_file": "/Path/To/ballot_manifest.xlsx", "sample_file": "/Path/To/sample.csv", "mvr_file": "/Path/To/mvr.json", "log_file": "/Path/To/log.json", "quantile": 0.8, "error_rate_1": 0.001, "error_rate_2": 0.0, "reps": 100, "max_cards": 293555, "strata": {"stratum_1": {"id": "stratum_1", "max_cards": 293555, "use_style": true, "replacement": false, "audit_type": "CARD_COMPARISON", "test": "alpha_mart", "estimator": "optimal_comparison", "bet": null, "test_kwargs": {}}}}, "contests": {"339": {"id": "339", "name": "2019 SF DA Contest", "risk_limit": 0.05, "cards": 146662, "choice_function": "IRV", "n_winners": 1, "share_to_win": null, "candidates": ["15", "16", "17", "18"], "winner": ["15"], "assertion_file": "/Path/To/contest_339_assertions.json", "audit_type": "CARD_COMPARISON", "test": "alpha_mart", "g": 0.1, "estim": "optimal_comparison", "bet": null, "use_style": true, "assertions": {"15 v 18 elim 16 17": {"contest": "339", "winner": "15", "loser": "18", "p_value": 0.04994290479444443, "margin": 0.028923647570604505, "p_history_length": 207, "proved": true, "sample_size": 207, "assorter_upper_bound": 1}, "18 v 17 elim 15 16": {"contest": "339", "winner": "18", "loser": "17", "p_value": 0.008418653014509435, "margin": 0.045792366120740224, "p_history_length": 207, "proved": true, "sample_size": 130, "assorter_upper_bound": 1}, "17 v 16 elim 15": {"contest": "339", "winner": "17", "loser": "16", "p_value": 0.0022794196455539545, "margin": 0.058079120699294995, "p_history_length": 207, "proved": true, "sample_size": 102, "assorter_upper_bound": 1}, "15 v 17 elim 16": {"contest": "339", "winner": "15", "loser": "17", "p_value": 0.00020251578032803977, "margin": 0.08064120222007065, "p_history_length": 207, "proved": true, "sample_size": 73, "assorter_upper_bound": 1}, "18 v 16 elim 15 17": {"contest": "339", "winner": "18", "loser": "16", "p_value": 0.00015696802816059395, "margin": 0.0830003681935334, "p_history_length": 207, "proved": true, "sample_size": 71, "assorter_upper_bound": 1}, "15 v 17 elim 16 18": {"contest": "339", "winner": "15", "loser": "17", "p_value": 8.763652687128994e-06, "margin": 0.10951712099930444, "p_history_length": 207, "proved": true, "sample_size": 54, "assorter_upper_bound": 1}, "15 v 16 elim 17 18": {"contest": "339", "winner": "15", "loser": "16", "p_value": 4.4478074219155553e-07, "margin": 0.1365247985163165, "p_history_length": 207, "proved": true, "sample_size": 43, "assorter_upper_bound": 1}, "15 v 16 elim 17": {"contest": "339", "winner": "15", "loser": "16", "p_value": 4.994560641272807e-07, "margin": 0.13548158350492967, "p_history_length": 207, "proved": true, "sample_size": 43, "assorter_upper_bound": 1}, "15 v 16 elim ": {"contest": "339", "winner": "15", "loser": "16", "p_value": 4.898533358103829e-08, "margin": 0.15626406294745743, "p_history_length": 207, "proved": true, "sample_size": 37, "assorter_upper_bound": 1}, "18 v 16 elim 15": {"contest": "339", "winner": "18", "loser": "16", "p_value": 1.1375482984733902e-07, "margin": 0.14875018750596603, "p_history_length": 207, "proved": true, "sample_size": 39, "assorter_upper_bound": 1}, "15 v 16 elim 18": {"contest": "339", "winner": "15", "loser": "16", "p_value": 1.5167265965762946e-08, "margin": 0.16666893946625572, "p_history_length": 207, "proved": true, "sample_size": 35, "assorter_upper_bound": 1}}, "tally": null, "sample_size": 207, "sample_threshold": 175011056564882680572169281431610914151877334350847425631070084166299710096, "assertion_json": [{"winner": "15", "loser": "18", "already_eliminated": ["16", "17"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 18]"}, {"winner": "18", "loser": "17", "already_eliminated": ["15", "16"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 18 17]"}, {"winner": "17", "loser": "16", "already_eliminated": ["15"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 17 16 18]"}, {"winner": "15", "loser": "17", "already_eliminated": ["16"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 17 18]"}, {"winner": "18", "loser": "16", "already_eliminated": ["15", "17"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 18 16]"}, {"winner": "15", "loser": "17", "already_eliminated": ["16", "18"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 17]"}, {"winner": "15", "loser": "16", "already_eliminated": ["17", "18"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 16]"}, {"winner": "15", "loser": "16", "already_eliminated": ["17"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 16 18]"}, {"winner": "15", "loser": "16", "already_eliminated": "", "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 16 17 18]"}, {"winner": "18", "loser": "16", "already_eliminated": ["15"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 18 17 16]"}, {"winner": "15", "loser": "16", "already_eliminated": ["18"], "assertion_type": "IRV_ELIMINATION", "explanation": "Rules out outcomes with tail [... 15 17 16]"}], "cvrs": 146662, "margins": {"15 v 18 elim 16 17": 0.028923647570604505, "18 v 17 elim 15 16": 0.045792366120740224, "17 v 16 elim 15": 0.058079120699294995, "15 v 17 elim 16": 0.08064120222007065, "18 v 16 elim 15 17": 0.0830003681935334, "15 v 17 elim 16 18": 0.10951712099930444, "15 v 16 elim 17 18": 0.1365247985163165, "15 v 16 elim 17": 0.13548158350492967, "15 v 16 elim ": 0.15626406294745743, "18 v 16 elim 15": 0.14875018750596603, "15 v 16 elim 18": 0.16666893946625572}, "p_values": {"15 v 18 elim 16 17": 0.04994290479444443, "18 v 17 elim 15 16": 0.008418653014509435, "17 v 16 elim 15": 0.0022794196455539545, "15 v 17 elim 16": 0.00020251578032803977, "18 v 16 elim 15 17": 0.00015696802816059395, "15 v 17 elim 16 18": 8.763652687128994e-06, "15 v 16 elim 17 18": 4.4478074219155553e-07, "15 v 16 elim 17": 4.994560641272807e-07, "15 v 16 elim ": 4.898533358103829e-08, "18 v 16 elim 15": 1.1375482984733902e-07, "15 v 16 elim 18": 1.5167265965762946e-08}, "proved": {"15 v 18 elim 16 17": true, "18 v 17 elim 15 16": true, "17 v 16 elim 15": true, "15 v 17 elim 16": true, "18 v 16 elim 15 17": true, "15 v 17 elim 16 18": true, "15 v 16 elim 17 18": true, "15 v 16 elim 17": true, "15 v 16 elim ": true, "18 v 16 elim 15": true, "15 v 16 elim 18": true}, "max_p": 0.04994290479444443}}}