From 503ca3220944bd8a08c46d8540ad966b3b83383f Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 16:10:18 -0500 Subject: [PATCH 01/11] Modify trend report class to allow for more options (like explicitly-passed API keys and specific election dates) and to better handle draws for test trend data. --- elex/api/trends.py | 84 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/elex/api/trends.py b/elex/api/trends.py index e63a14d..045b7b1 100644 --- a/elex/api/trends.py +++ b/elex/api/trends.py @@ -56,61 +56,93 @@ class BaseTrendReport(utils.UnicodeMixin): office_code = None api_report_id = 'Trend / g / US' - def __init__(self, trend_file=None, testresults=False): + def __init__(self, **kwargs): if not self.office_code or not self.api_report_id: raise NotImplementedError - self.testresults = testresults - self.load_raw_data(self.office_code, trend_file) - self.parties = [] - self.output_parties() + self.testresults = kwargs.get('testresults', False) + + self.electiondate = kwargs.get('electiondate', None) + self.api_key = kwargs.get('api_key', None) + self.trendfile = kwargs.get('trend_file', None) + + self.load_raw_data() + + if self.raw_data is None: + # Should we raise an error here, rather than creating an object + # that contains no actual results? + self.parties = None + else: + self.parties = [] + self.output_parties() + + def format_api_request_params(self): + params = {} + + if self.api_key is not None: + params['apiKey'] = self.api_key - def load_raw_data(self, office_code, trend_file=None): + return params + + def load_raw_data(self): """ Gets underlying data lists we need for parsing. """ - if trend_file: - self.raw_data = self.get_ap_file(trend_file) + if self.trendfile: + self.raw_data = self.get_ap_file() else: self.raw_data = self.get_ap_report( - office_code, params={ - 'test': self.testresults + 'test': self.testresults, + **self.format_api_request_params(), } ) - def get_ap_file(self, path): + def get_ap_file(self): """ Get raw data file. """ - with open(path, 'r') as readfile: + with open(self.trendfile, 'r') as readfile: data = json.load(readfile) return data['trendtable'] - def get_ap_report(self, key, params={}): + def get_ap_report(self, params={}): """ - Given a report number and a key for indexing, returns a list - of delegate counts by party. Makes a request from the AP - using requests. Formats that request with env vars. + Given a report number, returns a list of counts by party. + Makes a request from the AP using requests. Formats that request + with env vars. """ reports = utils.get_reports(params=params) - report_id = self.get_report_id(reports, key) + report_id = self.get_report_id(reports) if report_id: - r = utils.api_request('/reports/{0}'.format(report_id)) + r = utils.api_request( + '/reports/{0}'.format(report_id), + **self.format_api_request_params() + ) return r.json()['trendtable'] - def get_report_id(self, reports, key): + def get_report_id(self, reports): """ Takes a delSuper or delSum as the argument and returns organization-specific report ID. """ - for report in reports: - if ( - key == self.office_code and - report.get('title') in [self.api_report_id, self.api_test_report_id] - ): - id = report.get('id').rsplit('/', 1)[-1] - return id + matching_reports = [ + report for report in reports if report.get('title') in [ + self.api_report_id, + self.api_test_report_id + ] + ] + + if self.electiondate: # Can also use the explicit 'if is not none'. + matching_reports = [ + report for report in matching_reports + if report.get('electionDate') == self.electiondate + ] + + if matching_reports: + id = matching_reports[0].get('id').rsplit('/', 1)[-1] + return id + return None def output_parties(self): From b4a55b1689a0dfc917c86984fc27969d8716cf4a Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 16:12:31 -0500 Subject: [PATCH 02/11] Add a CLI decorator that allows commands to accept -- but not require -- an election date argument. Factor out logic of attaching date to election via CLI, so it can be written once and then used in both date-required and date-allowed decorators. --- elex/cli/decorators.py | 47 +++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/elex/cli/decorators.py b/elex/cli/decorators.py index d9ed2e9..3f5c388 100644 --- a/elex/cli/decorators.py +++ b/elex/cli/decorators.py @@ -6,9 +6,42 @@ from xml.dom.minidom import parseString +def attach_election_date(app, wrapped_fn, callback_context): + """ + Attaches value of date argument to election model used in commands. + + Common to both `accept_date_argument` and `require_date_argument` + decorators (i.e., this is called whether the date argument is + mandatory or optional). + """ + try: + app.election.electiondate = parse_date(app.pargs.date[0]) + return wrapped_fn(callback_context) + except ValueError: + text = '{0} could not be recognized as a date.' + app.log.error(text.format(app.pargs.date[0])) + app.close(1) + + return wrapped_fn(callback_context) + + +def accept_date_argument(fn): + """ + Decorator that checks for optional date argument. + """ + @wraps(fn) + def decorated(self): + if len(self.app.pargs.date) and self.app.pargs.date[0]: + returned_value = attach_election_date(self.app, fn, self) + return returned_value + return fn(self) + + return decorated + + def require_date_argument(fn): """ - Decorator that checks for date argument. + Decorator that checks for required date argument. """ @wraps(fn) def decorated(self): @@ -16,17 +49,7 @@ def decorated(self): if self.app.pargs.data_file: return fn(self) elif len(self.app.pargs.date) and self.app.pargs.date[0]: - try: - self.app.election.electiondate = parse_date( - self.app.pargs.date[0] - ) - return fn(self) - except ValueError: - text = '{0} could not be recognized as a date.' - self.app.log.error(text.format(self.app.pargs.date[0])) - self.app.close(1) - - return fn(self) + return attach_election_date(self.app, fn, self) else: text = 'No election date (e.g. `elex {0} 2015-11-\ 03`) or data file (e.g. `elex {0} --data-file path/to/file.json`) specified.' From d75b58b7dbc3d6530ab975de47e25970fca3281d Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 16:16:33 -0500 Subject: [PATCH 03/11] Modify house-trends, senate-trends and governor-trends CLI commands so they now consider API key, election date (via the newly-added decorator) and test-result arguments. Change how the trend-file argument gets passed to these. --- elex/cli/app.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/elex/cli/app.py b/elex/cli/app.py index 404fa94..3b4553b 100644 --- a/elex/cli/app.py +++ b/elex/cli/app.py @@ -3,7 +3,7 @@ from cement.ext.ext_logging import LoggingLogHandler from elex.api import Elections, DelegateReport, USGovernorTrendReport, USHouseTrendReport, USSenateTrendReport from elex.cli.constants import BANNER, LOG_FORMAT -from elex.cli.decorators import require_date_argument, require_ap_api_key +from elex.cli.decorators import accept_date_argument, require_date_argument, require_ap_api_key from elex.cli.hooks import add_election_hook, cachecontrol_logging_hook from shutil import rmtree @@ -375,6 +375,7 @@ def delegates(self): @expose(help="Get governor trend report") @require_ap_api_key + @accept_date_argument def governor_trends(self): """ ``elex governor-trends`` @@ -395,11 +396,35 @@ def governor_trends(self): Dem,Governor,7,7,12,19,20,0,-1,0 """ self.app.log.info('Getting governor trend report') - report = USGovernorTrendReport(self.app.pargs.trend_file) + + report_params = { + 'testresults': self.app.pargs.test is True, + } + + if self.app.election.electiondate is not None: + if self.app.pargs.test is True: + self.app.log.info( + 'Fetching test report for election {0}'.format( + self.app.election.electiondate + ) + ) + else: + self.app.log.info('Fetching report for election {0}'.format( + self.app.election.electiondate + )) + + report_params['electiondate'] = self.app.election.electiondate + + if self.app.pargs.trend_file is not None: + report_params['trend_file'] = self.app.pargs.trend_file + + report = USGovernorTrendReport(**report_params) + self.app.render(report.parties) @expose(help="Get US House trend report") @require_ap_api_key + @accept_date_argument def house_trends(self): """ ``elex house-trends`` @@ -420,11 +445,35 @@ def house_trends(self): Dem,U.S. House,201,201,0,201,193,0,+8,0 """ self.app.log.info('Getting US House trend report') - report = USHouseTrendReport(self.app.pargs.trend_file) + + report_params = { + 'testresults': self.app.pargs.test is True, + } + + if self.app.election.electiondate is not None: + if self.app.pargs.test is True: + self.app.log.info( + 'Fetching test report for election {0}'.format( + self.app.election.electiondate + ) + ) + else: + self.app.log.info('Fetching report for election {0}'.format( + self.app.election.electiondate + )) + + report_params['electiondate'] = self.app.election.electiondate + + if self.app.pargs.trend_file is not None: + report_params['trend_file'] = self.app.pargs.trend_file + + report = USHouseTrendReport(**report_params) + self.app.render(report.parties) @expose(help="Get US Senate trend report") @require_ap_api_key + @accept_date_argument def senate_trends(self): """ ``elex senate-trends`` @@ -445,7 +494,30 @@ def senate_trends(self): Dem,U.S. Senate,23,23,30,53,51,0,+2,0 """ self.app.log.info('Getting US Senate trend report') - report = USSenateTrendReport(self.app.pargs.trend_file) + + report_params = { + 'testresults': self.app.pargs.test is True, + } + + if self.app.election.electiondate is not None: + if self.app.pargs.test is True: + self.app.log.info( + 'Fetching test report for election {0}'.format( + self.app.election.electiondate + ) + ) + else: + self.app.log.info('Fetching report for election {0}'.format( + self.app.election.electiondate + )) + + report_params['electiondate'] = self.app.election.electiondate + + if self.app.pargs.trend_file is not None: + report_params['trend_file'] = self.app.pargs.trend_file + + report = USSenateTrendReport(**report_params) + self.app.render(report.parties) @expose(help="Get the next election (if date is specified, will be \ From 5e71d842c35b7cc2f72cf873da8f13d8fbb14c94 Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 16:54:27 -0500 Subject: [PATCH 04/11] Add backwards-compatibility for trend report classes instantiated with positional args (named args already worked). --- elex/api/trends.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/elex/api/trends.py b/elex/api/trends.py index 045b7b1..eccf08d 100644 --- a/elex/api/trends.py +++ b/elex/api/trends.py @@ -56,15 +56,20 @@ class BaseTrendReport(utils.UnicodeMixin): office_code = None api_report_id = 'Trend / g / US' - def __init__(self, **kwargs): + def __init__(self, *args, **kwargs): if not self.office_code or not self.api_report_id: raise NotImplementedError - self.testresults = kwargs.get('testresults', False) + if len(args): + # Shim to support former method signature. + defaults['trendfile'] = args[0] if len(args) > 0 else None + defaults['testresults'] = args[1] if len(args) > 1 else False + + self.testresults = kwargs.get('testresults', defaults['testresults']) self.electiondate = kwargs.get('electiondate', None) self.api_key = kwargs.get('api_key', None) - self.trendfile = kwargs.get('trend_file', None) + self.trendfile = kwargs.get('trend_file', defaults['trendfile']) self.load_raw_data() From ba90a50a9fc4c81c7a1736ba79e0379ae55e9c6b Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 17:03:03 -0500 Subject: [PATCH 05/11] Add and modify docstrings for some elex.api.trends methods. --- elex/api/trends.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/elex/api/trends.py b/elex/api/trends.py index eccf08d..1c95ecd 100644 --- a/elex/api/trends.py +++ b/elex/api/trends.py @@ -82,6 +82,10 @@ def __init__(self, *args, **kwargs): self.output_parties() def format_api_request_params(self): + """ + Sets params for both fetches in a given trend-report download + (i.e., list of reports and detail page). + """ params = {} if self.api_key is not None: @@ -114,6 +118,7 @@ def get_ap_file(self): def get_ap_report(self, params={}): """ Given a report number, returns a list of counts by party. + Makes a request from the AP using requests. Formats that request with env vars. """ @@ -128,8 +133,13 @@ def get_ap_report(self, params={}): def get_report_id(self, reports): """ - Takes a delSuper or delSum as the argument and returns - organization-specific report ID. + Narrows a list of all reports to just the type (U.S. House/U.S. + Senate/Governorships) specified in the subclass. + + If an election date was specified, limits results to those on + that day. Otherwise, finds the overall most recent report. + + Returns the versioned report ID where one exists. """ matching_reports = [ report for report in reports if report.get('title') in [ From 87355f644889aa346ff9ded4d3d8dfb747c04a0f Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 18:13:01 -0500 Subject: [PATCH 06/11] Linting and compatibility fixes so 'make test' works across Python 2.7, Python 3.6 and PyPy. --- elex/api/trends.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/elex/api/trends.py b/elex/api/trends.py index 1c95ecd..b954aca 100644 --- a/elex/api/trends.py +++ b/elex/api/trends.py @@ -60,10 +60,11 @@ def __init__(self, *args, **kwargs): if not self.office_code or not self.api_report_id: raise NotImplementedError - if len(args): - # Shim to support former method signature. - defaults['trendfile'] = args[0] if len(args) > 0 else None - defaults['testresults'] = args[1] if len(args) > 1 else False + # Shim to support former method signature. + defaults = { + 'trendfile': args[0] if len(args) > 0 else None, + 'testresults': args[1] if len(args) > 1 else False, + } self.testresults = kwargs.get('testresults', defaults['testresults']) @@ -100,12 +101,10 @@ def load_raw_data(self): if self.trendfile: self.raw_data = self.get_ap_file() else: - self.raw_data = self.get_ap_report( - params={ - 'test': self.testresults, - **self.format_api_request_params(), - } - ) + report_params = self.format_api_request_params() + report_params['test'] = self.testresults + + self.raw_data = self.get_ap_report(params=report_params) def get_ap_file(self): """ From 8681bd7dda5edc1dbe5473da33c7fabd936f5fa1 Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 18:14:45 -0500 Subject: [PATCH 07/11] Properly handle case where no matching reports are found in trend-report CLI commands. --- elex/cli/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elex/cli/app.py b/elex/cli/app.py index 3b4553b..4efb0ba 100644 --- a/elex/cli/app.py +++ b/elex/cli/app.py @@ -420,7 +420,7 @@ def governor_trends(self): report = USGovernorTrendReport(**report_params) - self.app.render(report.parties) + self.app.render(report.parties if report.parties is not None else []) @expose(help="Get US House trend report") @require_ap_api_key @@ -469,7 +469,7 @@ def house_trends(self): report = USHouseTrendReport(**report_params) - self.app.render(report.parties) + self.app.render(report.parties if report.parties is not None else []) @expose(help="Get US Senate trend report") @require_ap_api_key @@ -518,7 +518,7 @@ def senate_trends(self): report = USSenateTrendReport(**report_params) - self.app.render(report.parties) + self.app.render(report.parties if report.parties is not None else []) @expose(help="Get the next election (if date is specified, will be \ relative to that date, otherwise will use today's date)") From 3b18d645c2c2db09867b72c0fcb3925e87cd623d Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Tue, 2 Oct 2018 18:16:18 -0500 Subject: [PATCH 08/11] Modify newly-added optional date CLI command decorator to ignore date if the '-d/--data-file' argument is also set. This is identical to how the required-date decorator works. --- elex/cli/decorators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elex/cli/decorators.py b/elex/cli/decorators.py index 3f5c388..3a71483 100644 --- a/elex/cli/decorators.py +++ b/elex/cli/decorators.py @@ -31,7 +31,9 @@ def accept_date_argument(fn): """ @wraps(fn) def decorated(self): - if len(self.app.pargs.date) and self.app.pargs.date[0]: + if self.app.pargs.data_file: + return fn(self) + elif len(self.app.pargs.date) and self.app.pargs.date[0]: returned_value = attach_election_date(self.app, fn, self) return returned_value return fn(self) From b860cebf80473898475f47c283071b9d823500ff Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Thu, 4 Oct 2018 21:53:21 -0500 Subject: [PATCH 09/11] Add test data for balance-of-power trend reports. --- tests/data/00000000_trend_report_list.json | 1 + tests/data/20121106_gov_trends.json | 1 + tests/data/20121106_house_trends.json | 1 + tests/data/20121106_senate_trends.json | 1 + tests/data/20161108_gov_trends.json | 1 + tests/data/20161108_house_trends.json | 1 + tests/data/20161108_senate_trends.json | 1 + tests/data/20161210_gov_trends.json | 1 + tests/data/20161210_house_trends.json | 1 + tests/data/20161210_senate_trends.json | 1 + 10 files changed, 10 insertions(+) create mode 100644 tests/data/00000000_trend_report_list.json create mode 100644 tests/data/20121106_gov_trends.json create mode 100644 tests/data/20121106_house_trends.json create mode 100644 tests/data/20121106_senate_trends.json create mode 100644 tests/data/20161108_gov_trends.json create mode 100644 tests/data/20161108_house_trends.json create mode 100644 tests/data/20161108_senate_trends.json create mode 100644 tests/data/20161210_gov_trends.json create mode 100644 tests/data/20161210_house_trends.json create mode 100644 tests/data/20161210_senate_trends.json diff --git a/tests/data/00000000_trend_report_list.json b/tests/data/00000000_trend_report_list.json new file mode 100644 index 0000000..8411f84 --- /dev/null +++ b/tests/data/00000000_trend_report_list.json @@ -0,0 +1 @@ +{"id":"https://api.ap.org/v2/reports","title":"Election Reports","updated":"2018-10-04T22:28:41.750Z","reports":[{"id":"https://api.ap.org/v2/reports/3246ca5e0bb9475ca1dcd9438770319b","title":"EstimatedVotePercentage / AL","updated":"2017-08-16T14:53:27Z","electionDate":"2017-08-15","categories":[{"term":"EstimatedVotePercentage","scheme":"https://api.ap.org/cv/type/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"AL","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/3246ca5e0bb9475ca1dcd9438770319b"},{"id":"https://api.ap.org/v2/reports/89330c4d7d8644a38da850ac82e852c4","title":"PresCountyResults / US","updated":"2017-07-12T16:12:56.703Z","electionDate":"2016-11-08","categories":[{"term":"PresCountyResults","scheme":"https://api.ap.org/cv/type/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/89330c4d7d8644a38da850ac82e852c4","contentType":"application/csv"},{"id":"https://api.ap.org/v2/reports/4e56eb32d8bb4ee4bb92562e8108b201","title":"Pres / pres_summary / US","updated":"2017-02-17T12:46:00Z","electionDate":"2016-11-08","categories":[{"term":"Pres","scheme":"https://api.ap.org/cv/type/"},{"term":"pres_summary","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/4e56eb32d8bb4ee4bb92562e8108b201"},{"id":"https://api.ap.org/v2/reports/e23102cad58e45dc952532786a7f23b1","title":"Pres / pres_summary_all / US","updated":"2017-02-17T12:46:00Z","electionDate":"2016-11-08","categories":[{"term":"Pres","scheme":"https://api.ap.org/cv/type/"},{"term":"pres_summary_all","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/e23102cad58e45dc952532786a7f23b1"},{"id":"https://api.ap.org/v2/reports/55ff3ac245d94f148b91fc636c912ac2","title":"Pres / statebystate_pres / US","updated":"2017-02-17T12:46:00Z","electionDate":"2016-11-08","categories":[{"term":"Pres","scheme":"https://api.ap.org/cv/type/"},{"term":"statebystate_pres","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/55ff3ac245d94f148b91fc636c912ac2"},{"id":"https://api.ap.org/v2/reports/5913efb0d26041c29db4e5e9de4f23d4","title":"Trend / g / US","updated":"2016-12-10T22:41:44Z","electionDate":"2016-12-10","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"g","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/5913efb0d26041c29db4e5e9de4f23d4"},{"id":"https://api.ap.org/v2/reports/54d6adeb12c14c948f84598cbccb6f84","title":"Trend / h / US","updated":"2016-12-10T22:41:44Z","electionDate":"2016-12-10","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"h","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/54d6adeb12c14c948f84598cbccb6f84"},{"id":"https://api.ap.org/v2/reports/f5afcfd0b6fe4dfc9a937e46939b7fde","title":"Trend / s / US","updated":"2016-12-10T22:41:44Z","electionDate":"2016-12-10","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"s","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/f5afcfd0b6fe4dfc9a937e46939b7fde"},{"id":"https://api.ap.org/v2/reports/cf1f265d59e84fc8878bea929483d5f5","title":"Trend / h / US","updated":"2016-12-06T15:00:18Z","electionDate":"2016-11-08","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"h","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/cf1f265d59e84fc8878bea929483d5f5"},{"id":"https://api.ap.org/v2/reports/c31e3ee20ddd4854803f196a4f3a2321","title":"Trend / g / US","updated":"2016-12-06T13:00:36Z","electionDate":"2016-11-08","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"g","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/c31e3ee20ddd4854803f196a4f3a2321"},{"id":"https://api.ap.org/v2/reports/9f8b8120732844a8853dd84ad13b8735","title":"Trend / s / US","updated":"2016-11-28T14:42:26Z","electionDate":"2016-11-08","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"s","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/9f8b8120732844a8853dd84ad13b8735"},{"id":"https://api.ap.org/v2/reports/7f9736cbb85643be900677f08f0880fb","title":"Trend / g / US","updated":"2016-08-18T17:21:20.537Z","electionDate":"2012-11-06","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"g","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/7f9736cbb85643be900677f08f0880fb"},{"id":"https://api.ap.org/v2/reports/79a6d01ce88f4efca7bf954d3c98d1ef","title":"Trend / h / US","updated":"2016-08-18T17:21:16.747Z","electionDate":"2012-11-06","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"h","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/79a6d01ce88f4efca7bf954d3c98d1ef"},{"id":"https://api.ap.org/v2/reports/2831ab7043e24c68875d213cd5bf1d3e","title":"Trend / s / US","updated":"2016-08-18T17:21:13.410Z","electionDate":"2012-11-06","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"s","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/2831ab7043e24c68875d213cd5bf1d3e"},{"id":"https://api.ap.org/v2/reports/a29891db061e41bc88eca7336f45255e","title":"Delegates / delstate","updated":"2016-07-25T11:33:08Z","categories":[{"term":"Delegates","scheme":"https://api.ap.org/cv/type/"},{"term":"delstate","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"}],"contentLink":"https://api.ap.org/v2/reports/a29891db061e41bc88eca7336f45255e"},{"id":"https://api.ap.org/v2/reports/b0a73da9e75a471d811597ed733b21eb","title":"Delegates / delsum","updated":"2016-07-25T11:33:08Z","categories":[{"term":"Delegates","scheme":"https://api.ap.org/cv/type/"},{"term":"delsum","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"}],"contentLink":"https://api.ap.org/v2/reports/b0a73da9e75a471d811597ed733b21eb"},{"id":"https://api.ap.org/v2/reports/0ad58ba43f8c4ffabdb23f008669a7cf","title":"Delegates / delsuper","updated":"2016-07-25T11:33:08Z","categories":[{"term":"Delegates","scheme":"https://api.ap.org/cv/type/"},{"term":"delsuper","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"}],"contentLink":"https://api.ap.org/v2/reports/0ad58ba43f8c4ffabdb23f008669a7cf"}],"nextrequest":"https://api.ap.org/v2/reports?format=JSON&minDateTime=2018-09-01T16%3a00%3a00.000Z"} diff --git a/tests/data/20121106_gov_trends.json b/tests/data/20121106_gov_trends.json new file mode 100644 index 0000000..a328553 --- /dev/null +++ b/tests/data/20121106_gov_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "Governor", "OfficeTypeCode": "G", "Test": "0", "timestamp": "2012-11-29T15:59:16Z", "party": [{"title": "Dem", "trend": [{"Won": "7"}, {"Leading": "0"}, {"Holdovers": "12"}, {"Winning Trend": "19"}, {"Current": "20"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-1"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "4"}, {"Leading": "0"}, {"Holdovers": "26"}, {"Winning Trend": "30"}, {"Current": "29"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+1"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "1"}, {"Winning Trend": "1"}, {"Current": "1"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20121106_house_trends.json b/tests/data/20121106_house_trends.json new file mode 100644 index 0000000..9882214 --- /dev/null +++ b/tests/data/20121106_house_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "U.S. House", "OfficeTypeCode": "H", "Test": "0", "timestamp": "2012-12-10T21:10:07Z", "party": [{"title": "Dem", "trend": [{"Won": "201"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "201"}, {"Current": "193"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+8"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "234"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "234"}, {"Current": "242"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-8"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "0"}, {"Current": "0"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20121106_senate_trends.json b/tests/data/20121106_senate_trends.json new file mode 100644 index 0000000..7e699c5 --- /dev/null +++ b/tests/data/20121106_senate_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "U.S. Senate", "OfficeTypeCode": "S", "Test": "0", "timestamp": "2012-11-29T15:59:17Z", "party": [{"title": "Dem", "trend": [{"Won": "23"}, {"Leading": "0"}, {"Holdovers": "30"}, {"Winning Trend": "53"}, {"Current": "51"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+2"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "8"}, {"Leading": "0"}, {"Holdovers": "37"}, {"Winning Trend": "45"}, {"Current": "47"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-2"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "2"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "2"}, {"Current": "2"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20161108_gov_trends.json b/tests/data/20161108_gov_trends.json new file mode 100644 index 0000000..21c2cb2 --- /dev/null +++ b/tests/data/20161108_gov_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "Governor", "OfficeTypeCode": "G", "Test": "0", "timestamp": "2016-12-06T16:13:19Z", "party": [{"title": "Dem", "trend": [{"Won": "6"}, {"Leading": "0"}, {"Holdovers": "10"}, {"Winning Trend": "16"}, {"Current": "18"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-2"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "6"}, {"Leading": "0"}, {"Holdovers": "27"}, {"Winning Trend": "33"}, {"Current": "31"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+2"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "1"}, {"Winning Trend": "1"}, {"Current": "1"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20161108_house_trends.json b/tests/data/20161108_house_trends.json new file mode 100644 index 0000000..c6bc2ea --- /dev/null +++ b/tests/data/20161108_house_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "U.S. House", "OfficeTypeCode": "H", "Test": "0", "timestamp": "2016-12-06T19:58:50Z", "party": [{"title": "Dem", "trend": [{"Won": "194"}, {"Leading": "1"}, {"Holdovers": "0"}, {"Winning Trend": "195"}, {"Current": "188"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+6"}, {"Leaders": "+1"}]}}, {"title": "GOP", "trend": [{"Won": "240"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "240"}, {"Current": "247"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-6"}, {"Leaders": "-1"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "0"}, {"Current": "0"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20161108_senate_trends.json b/tests/data/20161108_senate_trends.json new file mode 100644 index 0000000..c371af7 --- /dev/null +++ b/tests/data/20161108_senate_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "U.S. Senate", "OfficeTypeCode": "S", "Test": "0", "timestamp": "2016-11-28T19:40:38Z", "party": [{"title": "Dem", "trend": [{"Won": "12"}, {"Leading": "0"}, {"Holdovers": "34"}, {"Winning Trend": "46"}, {"Current": "44"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+2"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "21"}, {"Leading": "1"}, {"Holdovers": "30"}, {"Winning Trend": "52"}, {"Current": "54"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-2"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "2"}, {"Winning Trend": "2"}, {"Current": "2"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20161210_gov_trends.json b/tests/data/20161210_gov_trends.json new file mode 100644 index 0000000..19af63e --- /dev/null +++ b/tests/data/20161210_gov_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "Governor", "OfficeTypeCode": "G", "Test": "0", "timestamp": "2016-12-06T18:15:13Z", "party": [{"title": "Dem", "trend": [{"Won": "6"}, {"Leading": "0"}, {"Holdovers": "10"}, {"Winning Trend": "16"}, {"Current": "18"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-2"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "6"}, {"Leading": "0"}, {"Holdovers": "27"}, {"Winning Trend": "33"}, {"Current": "31"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+2"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "1"}, {"Winning Trend": "1"}, {"Current": "1"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20161210_house_trends.json b/tests/data/20161210_house_trends.json new file mode 100644 index 0000000..e291a9c --- /dev/null +++ b/tests/data/20161210_house_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "U.S. House", "OfficeTypeCode": "H", "Test": "0", "timestamp": "2016-12-11T03:18:54Z", "party": [{"title": "Dem", "trend": [{"Won": "194"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "194"}, {"Current": "188"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+6"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "241"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "241"}, {"Current": "247"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-6"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "0"}, {"Winning Trend": "0"}, {"Current": "0"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file diff --git a/tests/data/20161210_senate_trends.json b/tests/data/20161210_senate_trends.json new file mode 100644 index 0000000..d881948 --- /dev/null +++ b/tests/data/20161210_senate_trends.json @@ -0,0 +1 @@ +{ "trendtable": {"office": "U.S. Senate", "OfficeTypeCode": "S", "Test": "0", "timestamp": "2016-12-11T03:06:23Z", "party": [{"title": "Dem", "trend": [{"Won": "12"}, {"Leading": "0"}, {"Holdovers": "34"}, {"Winning Trend": "46"}, {"Current": "44"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "+2"}, {"Leaders": "0"}]}}, {"title": "GOP", "trend": [{"Won": "22"}, {"Leading": "0"}, {"Holdovers": "30"}, {"Winning Trend": "52"}, {"Current": "54"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "-2"}, {"Leaders": "0"}]}}, {"title": "Others", "trend": [{"Won": "0"}, {"Leading": "0"}, {"Holdovers": "2"}, {"Winning Trend": "2"}, {"Current": "2"}, {"InsufficientVote": "0"}], "NetChange": { "trend": [{"Winners": "0"}, {"Leaders": "0"}]}}], "InsufficientVote": { "trend": [{"Dem Open": "0"}, {"GOP Open": "0"}, {"Oth Open": "0"}, {"Dem Incumbent": "0"}, {"GOP Incumbent": "0"}, {"Oth Incumbent": "0"}, {"Total": "0"}]}}} \ No newline at end of file From 3e5cd36525e8e47672e8236d30fb2bf866d12286 Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Thu, 4 Oct 2018 21:54:09 -0500 Subject: [PATCH 10/11] Add 'mock' dev-dependency for Python < 3 (it's part of the standard library under 'unittest' in Python 3). --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index da77ae2..594350c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,5 @@ sphinx-rtd-theme==0.1.9 nose2 tox flake8 + +mock==2.0.0;python_version<'3.0' From 32ef77845625dc4a4e3c7cdd8ef03ef2babad44a Mon Sep 17 00:00:00 2001 From: Allan James Vestal Date: Thu, 4 Oct 2018 21:55:49 -0500 Subject: [PATCH 11/11] Add tests to ensure the trend-report downloader gets data for the correct election. Rename the test class contained in 'tests.test_trend_reports'. --- tests/test_trend_reports.py | 98 ++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/tests/test_trend_reports.py b/tests/test_trend_reports.py index 81d4c5c..d94df39 100644 --- a/tests/test_trend_reports.py +++ b/tests/test_trend_reports.py @@ -1,7 +1,80 @@ +import json import tests +from elex.api import USHouseTrendReport -class TestDelegateReports(tests.TrendReportTestCase): +try: + from unittest.mock import patch +except ImportError: + from mock import patch + + +TREND_REPORT_DATA_FILES = { + '5913efb0d26041c29db4e5e9de4f23d4': '20161210_gov_trends.json', + '54d6adeb12c14c948f84598cbccb6f84': '20161210_house_trends.json', + 'f5afcfd0b6fe4dfc9a937e46939b7fde': '20161210_senate_trends.json', + + 'cf1f265d59e84fc8878bea929483d5f5': '20161108_house_trends.json', + 'c31e3ee20ddd4854803f196a4f3a2321': '20161108_gov_trends.json', + '9f8b8120732844a8853dd84ad13b8735': '20161108_senate_trends.json', + + '7f9736cbb85643be900677f08f0880fb': '20121106_gov_trends.json', + '79a6d01ce88f4efca7bf954d3c98d1ef': '20121106_house_trends.json', + '2831ab7043e24c68875d213cd5bf1d3e': '20121106_senate_trends.json', +} + + +class MockedResponse(object): + def __init__(self, status=200, json_body={}): + self.ok = True if status == 200 else False + + self.json_body = json_body + + def json(self): + return self.json_body + + +def patched_api_request(url, **params): + """""" + if (url != '/reports'): + report_id = url.lstrip('/reports/') + report_filepath = 'tests/data/%s' % TREND_REPORT_DATA_FILES[report_id] + + with open(report_filepath, 'r') as report_file: + report_json = json.load(report_file) + + return MockedResponse(json_body=report_json) + + report_list_filepath = 'tests/data/00000000_trend_report_list.json' + + with open(report_list_filepath, 'r') as report_list_file: + report_list = json.load(report_list_file) + + return MockedResponse(json_body=report_list) + + +def compare_trend_reports(control_report, observed_report): + """ + Compare two trend reports' serialized dicts. + """ + control_parties = {_.party: _ for _ in control_report} + observed_parties = {_.party: _ for _ in observed_report} + + report_facets_match = [] + + for party_slug, control_party_obj in control_parties.items(): + control_party = control_party_obj.serialize() + observed_party = observed_parties[party_slug].serialize() + + for trend_facet in control_party.keys(): + report_facets_match.append( + control_party[trend_facet] == observed_party[trend_facet] + ) + + return all(report_facets_match) + + +class TestBalanceOfPowerReports(tests.TrendReportTestCase): """ @TODO Not very sufficient tests """ @@ -32,3 +105,26 @@ def test_us_governor_other_won(self): def test_us_governor_other_leading(self): trend = self.governor_trends.parties[2] self.assertEqual(trend.leading, '0') + + @patch('elex.api.utils.api_request', side_effect=patched_api_request) + def test_unset_date_gets_latest_matching_report(self, patched_fn): + control_trend_file = 'tests/data/20161210_house_trends.json' + control_trend = USHouseTrendReport(control_trend_file).parties + + observed_trend = USHouseTrendReport().parties + + self.assertTrue(compare_trend_reports(control_trend, observed_trend)) + + @patch('elex.api.utils.api_request', side_effect=patched_api_request) + def test_set_date_gets_correct_report(self, patched_fn): + control_trend_file = 'tests/data/20121106_house_trends.json' + control_trend = USHouseTrendReport(control_trend_file).parties + + mismatch_trend_file = 'tests/data/20161210_house_trends.json' + mismatch_trend = USHouseTrendReport(mismatch_trend_file).parties + + observed_trend = USHouseTrendReport(electiondate='2012-11-06').parties + + self.assertTrue(compare_trend_reports(control_trend, observed_trend)) + + self.assertFalse(compare_trend_reports(mismatch_trend, observed_trend))