Support the 'electiondate' argument when pulling trend reports

elex/api/trends.py

@@ -56,61 +56,107 @@ class BaseTrendReport(utils.UnicodeMixin):
     office_code = None
     api_report_id = 'Trend / g / US'
 
-    def __init__(self, trend_file=None, testresults=False):
+    def __init__(self, *args, **kwargs):
         if not self.office_code or not self.api_report_id:
             raise NotImplementedError
-        self.testresults = testresults
 
-        self.load_raw_data(self.office_code, trend_file)
-        self.parties = []
-        self.output_parties()
+        # Shim to support former method signature.
+        defaults = {
+            'trendfile': args[0] if len(args) > 0 else None,
+            'testresults': args[1] if len(args) > 1 else False,
+        }
 
-    def load_raw_data(self, office_code, trend_file=None):
+        self.testresults = kwargs.get('testresults', defaults['testresults'])
+
+        self.electiondate = kwargs.get('electiondate', None)
+        self.api_key = kwargs.get('api_key', None)
+        self.trendfile = kwargs.get('trend_file', defaults['trendfile'])
+
+        self.load_raw_data()
+
+        if self.raw_data is None:
+            # Should we raise an error here, rather than creating an object
+            # that contains no actual results?
+            self.parties = None
+        else:
+            self.parties = []
+            self.output_parties()
+
+    def format_api_request_params(self):
+        """
+        Sets params for both fetches in a given trend-report download
+        (i.e., list of reports and detail page).
+        """
+        params = {}
+
+        if self.api_key is not None:
+            params['apiKey'] = self.api_key
+
+        return params
+
+    def load_raw_data(self):
         """
         Gets underlying data lists we need for parsing.
         """
-        if trend_file:
-            self.raw_data = self.get_ap_file(trend_file)
+        if self.trendfile:
+            self.raw_data = self.get_ap_file()
         else:
-            self.raw_data = self.get_ap_report(
-                office_code,
-                params={
-                    'test': self.testresults
-                }
-            )
+            report_params = self.format_api_request_params()
+            report_params['test'] = self.testresults
 
-    def get_ap_file(self, path):
+            self.raw_data = self.get_ap_report(params=report_params)
+
+    def get_ap_file(self):
         """
         Get raw data file.
         """
-        with open(path, 'r') as readfile:
+        with open(self.trendfile, 'r') as readfile:
             data = json.load(readfile)
             return data['trendtable']
 
-    def get_ap_report(self, key, params={}):
+    def get_ap_report(self, params={}):
         """
-        Given a report number and a key for indexing, returns a list
-        of delegate counts by party. Makes a request from the AP
-        using requests. Formats that request with env vars.
+        Given a report number, returns a list of counts by party.
+
+        Makes a request from the AP using requests. Formats that request
+        with env vars.
         """
         reports = utils.get_reports(params=params)
-        report_id = self.get_report_id(reports, key)
+        report_id = self.get_report_id(reports)
         if report_id:
-            r = utils.api_request('/reports/{0}'.format(report_id))
+            r = utils.api_request(
+                '/reports/{0}'.format(report_id),
+                **self.format_api_request_params()
+            )
             return r.json()['trendtable']
 
-    def get_report_id(self, reports, key):
+    def get_report_id(self, reports):
         """
-        Takes a delSuper or delSum as the argument and returns
-        organization-specific report ID.
+        Narrows a list of all reports to just the type (U.S. House/U.S.
+        Senate/Governorships) specified in the subclass.
+
+        If an election date was specified, limits results to those on
+        that day. Otherwise, finds the overall most recent report.
+
+        Returns the versioned report ID where one exists.
         """
-        for report in reports:
-            if (
-                key == self.office_code and
-                report.get('title') in [self.api_report_id, self.api_test_report_id]
-            ):
-                id = report.get('id').rsplit('/', 1)[-1]
-                return id
+        matching_reports = [
+            report for report in reports if report.get('title') in [
+                self.api_report_id,
+                self.api_test_report_id
+            ]
+        ]
+
+        if self.electiondate:  # Can also use the explicit 'if is not none'.
+            matching_reports = [
+                report for report in matching_reports
+                if report.get('electionDate') == self.electiondate
+            ]
+
+        if matching_reports:
+            id = matching_reports[0].get('id').rsplit('/', 1)[-1]
+            return id
+
         return None
 
     def output_parties(self):

elex/cli/app.py

@@ -3,7 +3,7 @@
 from cement.ext.ext_logging import LoggingLogHandler
 from elex.api import Elections, DelegateReport, USGovernorTrendReport, USHouseTrendReport, USSenateTrendReport
 from elex.cli.constants import BANNER, LOG_FORMAT
-from elex.cli.decorators import require_date_argument, require_ap_api_key
+from elex.cli.decorators import accept_date_argument, require_date_argument, require_ap_api_key
 from elex.cli.hooks import add_election_hook, cachecontrol_logging_hook
 from shutil import rmtree
 
@@ -375,6 +375,7 @@ def delegates(self):
 
     @expose(help="Get governor trend report")
     @require_ap_api_key
+    @accept_date_argument
     def governor_trends(self):
         """
         ``elex governor-trends``
@@ -395,11 +396,35 @@ def governor_trends(self):
             Dem,Governor,7,7,12,19,20,0,-1,0
         """
         self.app.log.info('Getting governor trend report')
-        report = USGovernorTrendReport(self.app.pargs.trend_file)
-        self.app.render(report.parties)
+
+        report_params = {
+            'testresults': self.app.pargs.test is True,
+        }
+
+        if self.app.election.electiondate is not None:
+            if self.app.pargs.test is True:
+                self.app.log.info(
+                    'Fetching test report for election {0}'.format(
+                        self.app.election.electiondate
+                    )
+                )
+            else:
+                self.app.log.info('Fetching report for election {0}'.format(
+                    self.app.election.electiondate
+                ))
+
+            report_params['electiondate'] = self.app.election.electiondate
+
+        if self.app.pargs.trend_file is not None:
+            report_params['trend_file'] = self.app.pargs.trend_file
+
+        report = USGovernorTrendReport(**report_params)
+
+        self.app.render(report.parties if report.parties is not None else [])
 
     @expose(help="Get US House trend report")
     @require_ap_api_key
+    @accept_date_argument
     def house_trends(self):
         """
         ``elex house-trends``
@@ -420,11 +445,35 @@ def house_trends(self):
             Dem,U.S. House,201,201,0,201,193,0,+8,0
         """
         self.app.log.info('Getting US House trend report')
-        report = USHouseTrendReport(self.app.pargs.trend_file)
-        self.app.render(report.parties)
+
+        report_params = {
+            'testresults': self.app.pargs.test is True,
+        }
+
+        if self.app.election.electiondate is not None:
+            if self.app.pargs.test is True:
+                self.app.log.info(
+                    'Fetching test report for election {0}'.format(
+                        self.app.election.electiondate
+                    )
+                )
+            else:
+                self.app.log.info('Fetching report for election {0}'.format(
+                    self.app.election.electiondate
+                ))
+
+            report_params['electiondate'] = self.app.election.electiondate
+
+        if self.app.pargs.trend_file is not None:
+            report_params['trend_file'] = self.app.pargs.trend_file
+
+        report = USHouseTrendReport(**report_params)
+
+        self.app.render(report.parties if report.parties is not None else [])
 
     @expose(help="Get US Senate trend report")
     @require_ap_api_key
+    @accept_date_argument
     def senate_trends(self):
         """
         ``elex senate-trends``
@@ -445,8 +494,31 @@ def senate_trends(self):
             Dem,U.S. Senate,23,23,30,53,51,0,+2,0
         """
         self.app.log.info('Getting US Senate trend report')
-        report = USSenateTrendReport(self.app.pargs.trend_file)
-        self.app.render(report.parties)
+
+        report_params = {
+            'testresults': self.app.pargs.test is True,
+        }
+
+        if self.app.election.electiondate is not None:
+            if self.app.pargs.test is True:
+                self.app.log.info(
+                    'Fetching test report for election {0}'.format(
+                        self.app.election.electiondate
+                    )
+                )
+            else:
+                self.app.log.info('Fetching report for election {0}'.format(
+                    self.app.election.electiondate
+                ))
+
+            report_params['electiondate'] = self.app.election.electiondate
+
+        if self.app.pargs.trend_file is not None:
+            report_params['trend_file'] = self.app.pargs.trend_file
+
+        report = USSenateTrendReport(**report_params)
+
+        self.app.render(report.parties if report.parties is not None else [])
 
     @expose(help="Get the next election (if date is specified, will be \
 relative to that date, otherwise will use today's date)")

elex/cli/decorators.py

@@ -6,27 +6,52 @@
 from xml.dom.minidom import parseString
 
 
-def require_date_argument(fn):
+def attach_election_date(app, wrapped_fn, callback_context):
+    """
+    Attaches value of date argument to election model used in commands.
+
+    Common to both `accept_date_argument` and `require_date_argument`
+    decorators (i.e., this is called whether the date argument is
+    mandatory or optional).
+    """
+    try:
+        app.election.electiondate = parse_date(app.pargs.date[0])
+        return wrapped_fn(callback_context)
+    except ValueError:
+        text = '{0} could not be recognized as a date.'
+        app.log.error(text.format(app.pargs.date[0]))
+        app.close(1)
+
+    return wrapped_fn(callback_context)
+
+
+def accept_date_argument(fn):
     """
-    Decorator that checks for date argument.
+    Decorator that checks for optional date argument.
     """
     @wraps(fn)
     def decorated(self):
-        name = fn.__name__.replace('_', '-')
         if self.app.pargs.data_file:
             return fn(self)
         elif len(self.app.pargs.date) and self.app.pargs.date[0]:
-            try:
-                self.app.election.electiondate = parse_date(
-                    self.app.pargs.date[0]
-                )
-                return fn(self)
-            except ValueError:
-                text = '{0} could not be recognized as a date.'
-                self.app.log.error(text.format(self.app.pargs.date[0]))
-                self.app.close(1)
+            returned_value = attach_election_date(self.app, fn, self)
+            return returned_value
+        return fn(self)
+
+    return decorated
 
+
+def require_date_argument(fn):
+    """
+    Decorator that checks for required date argument.
+    """
+    @wraps(fn)
+    def decorated(self):
+        name = fn.__name__.replace('_', '-')
+        if self.app.pargs.data_file:
             return fn(self)
+        elif len(self.app.pargs.date) and self.app.pargs.date[0]:
+            return attach_election_date(self.app, fn, self)
         else:
             text = 'No election date (e.g. `elex {0} 2015-11-\
 03`) or data file (e.g. `elex {0} --data-file path/to/file.json`) specified.'

requirements-dev.txt

@@ -4,3 +4,5 @@ sphinx-rtd-theme==0.1.9
 nose2
 tox
 flake8
+
+mock==2.0.0;python_version<'3.0'

tests/data/00000000_trend_report_list.json

@@ -0,0 +1 @@
+{"id":"https://api.ap.org/v2/reports","title":"Election Reports","updated":"2018-10-04T22:28:41.750Z","reports":[{"id":"https://api.ap.org/v2/reports/3246ca5e0bb9475ca1dcd9438770319b","title":"EstimatedVotePercentage / AL","updated":"2017-08-16T14:53:27Z","electionDate":"2017-08-15","categories":[{"term":"EstimatedVotePercentage","scheme":"https://api.ap.org/cv/type/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"AL","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/3246ca5e0bb9475ca1dcd9438770319b"},{"id":"https://api.ap.org/v2/reports/89330c4d7d8644a38da850ac82e852c4","title":"PresCountyResults / US","updated":"2017-07-12T16:12:56.703Z","electionDate":"2016-11-08","categories":[{"term":"PresCountyResults","scheme":"https://api.ap.org/cv/type/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/89330c4d7d8644a38da850ac82e852c4","contentType":"application/csv"},{"id":"https://api.ap.org/v2/reports/4e56eb32d8bb4ee4bb92562e8108b201","title":"Pres / pres_summary / US","updated":"2017-02-17T12:46:00Z","electionDate":"2016-11-08","categories":[{"term":"Pres","scheme":"https://api.ap.org/cv/type/"},{"term":"pres_summary","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/4e56eb32d8bb4ee4bb92562e8108b201"},{"id":"https://api.ap.org/v2/reports/e23102cad58e45dc952532786a7f23b1","title":"Pres / pres_summary_all / US","updated":"2017-02-17T12:46:00Z","electionDate":"2016-11-08","categories":[{"term":"Pres","scheme":"https://api.ap.org/cv/type/"},{"term":"pres_summary_all","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/e23102cad58e45dc952532786a7f23b1"},{"id":"https://api.ap.org/v2/reports/55ff3ac245d94f148b91fc636c912ac2","title":"Pres / statebystate_pres / US","updated":"2017-02-17T12:46:00Z","electionDate":"2016-11-08","categories":[{"term":"Pres","scheme":"https://api.ap.org/cv/type/"},{"term":"statebystate_pres","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/55ff3ac245d94f148b91fc636c912ac2"},{"id":"https://api.ap.org/v2/reports/5913efb0d26041c29db4e5e9de4f23d4","title":"Trend / g / US","updated":"2016-12-10T22:41:44Z","electionDate":"2016-12-10","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"g","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/5913efb0d26041c29db4e5e9de4f23d4"},{"id":"https://api.ap.org/v2/reports/54d6adeb12c14c948f84598cbccb6f84","title":"Trend / h / US","updated":"2016-12-10T22:41:44Z","electionDate":"2016-12-10","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"h","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/54d6adeb12c14c948f84598cbccb6f84"},{"id":"https://api.ap.org/v2/reports/f5afcfd0b6fe4dfc9a937e46939b7fde","title":"Trend / s / US","updated":"2016-12-10T22:41:44Z","electionDate":"2016-12-10","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"s","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/f5afcfd0b6fe4dfc9a937e46939b7fde"},{"id":"https://api.ap.org/v2/reports/cf1f265d59e84fc8878bea929483d5f5","title":"Trend / h / US","updated":"2016-12-06T15:00:18Z","electionDate":"2016-11-08","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"h","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/cf1f265d59e84fc8878bea929483d5f5"},{"id":"https://api.ap.org/v2/reports/c31e3ee20ddd4854803f196a4f3a2321","title":"Trend / g / US","updated":"2016-12-06T13:00:36Z","electionDate":"2016-11-08","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"g","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/c31e3ee20ddd4854803f196a4f3a2321"},{"id":"https://api.ap.org/v2/reports/9f8b8120732844a8853dd84ad13b8735","title":"Trend / s / US","updated":"2016-11-28T14:42:26Z","electionDate":"2016-11-08","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"s","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/9f8b8120732844a8853dd84ad13b8735"},{"id":"https://api.ap.org/v2/reports/7f9736cbb85643be900677f08f0880fb","title":"Trend / g / US","updated":"2016-08-18T17:21:20.537Z","electionDate":"2012-11-06","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"g","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/7f9736cbb85643be900677f08f0880fb"},{"id":"https://api.ap.org/v2/reports/79a6d01ce88f4efca7bf954d3c98d1ef","title":"Trend / h / US","updated":"2016-08-18T17:21:16.747Z","electionDate":"2012-11-06","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"h","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/79a6d01ce88f4efca7bf954d3c98d1ef"},{"id":"https://api.ap.org/v2/reports/2831ab7043e24c68875d213cd5bf1d3e","title":"Trend / s / US","updated":"2016-08-18T17:21:13.410Z","electionDate":"2012-11-06","categories":[{"term":"Trend","scheme":"https://api.ap.org/cv/type/"},{"term":"s","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"},{"term":"US","scheme":"https://api.ap.org/cv/geo/"}],"contentLink":"https://api.ap.org/v2/reports/2831ab7043e24c68875d213cd5bf1d3e"},{"id":"https://api.ap.org/v2/reports/a29891db061e41bc88eca7336f45255e","title":"Delegates / delstate","updated":"2016-07-25T11:33:08Z","categories":[{"term":"Delegates","scheme":"https://api.ap.org/cv/type/"},{"term":"delstate","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"}],"contentLink":"https://api.ap.org/v2/reports/a29891db061e41bc88eca7336f45255e"},{"id":"https://api.ap.org/v2/reports/b0a73da9e75a471d811597ed733b21eb","title":"Delegates / delsum","updated":"2016-07-25T11:33:08Z","categories":[{"term":"Delegates","scheme":"https://api.ap.org/cv/type/"},{"term":"delsum","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"}],"contentLink":"https://api.ap.org/v2/reports/b0a73da9e75a471d811597ed733b21eb"},{"id":"https://api.ap.org/v2/reports/0ad58ba43f8c4ffabdb23f008669a7cf","title":"Delegates / delsuper","updated":"2016-07-25T11:33:08Z","categories":[{"term":"Delegates","scheme":"https://api.ap.org/cv/type/"},{"term":"delsuper","scheme":"https://api.ap.org/cv/subtype/"},{"term":"false","scheme":"https://api.ap.org/cv/test/"}],"contentLink":"https://api.ap.org/v2/reports/0ad58ba43f8c4ffabdb23f008669a7cf"}],"nextrequest":"https://api.ap.org/v2/reports?format=JSON&minDateTime=2018-09-01T16%3a00%3a00.000Z"}