Merge pull request #41 from EFForg/openstates_v3_upgrade

Openstates v3 upgrade

Dockerfile

@@ -1,7 +1,7 @@
-FROM python:2.7-stretch
+FROM python:2.7-buster
 RUN apt-get update && \
   curl -sL https://deb.nodesource.com/setup_4.x | bash && \
-  apt-get -y install git uwsgi libpq-dev curl unzip nodejs
+  apt-get -y install git uwsgi libpq-dev curl unzip nodejs npm
 
 RUN  mkdir /ngrok && \
      cd /ngrok && \

INSTALLATION.md

@@ -10,7 +10,7 @@ At a minimum, you will need to set:
 
 * SECRET_KEY, to secure login sessions cryptographically
     * This will be created for you automatically if you use the deploy to Heroku button, or you can generate one using with this Javascript one-liner: `chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-=!@#$%^&*()_+:<>{}[]".split(''); key = ''; for (i = 0; i < 50; i++) key += chars[Math.floor(Math.random() * chars.length)]; alert(key);`
-* SUNLIGHT_API_KEY, to do Congressional lookups. Sign up for one at [SunlightFoundation.com](https://sunlightfoundation.com/api/accounts/register/)
+* OPENSTATES_API_KEY, to do Congressional lookups. Sign up for one at [OpenStates.org](https://openstates.org/accounts/signup/)
 * TWILIO_ACCOUNT_SID, for an account with at least one purchased phone number
 * TWILIO_AUTH_TOKEN, for the same account
 * INSTALLED_ORG, displayed on the site homepage
@@ -66,8 +66,8 @@ To install locally and run in debug mode use:
     # create an admin user
     python manager.py createadminuser
 
-    # if testing twilio, run in another tab
-    ngrok http 5000
+    # if testing twilio, run in another tab -- you will have to sign up for an ngrok account to get your auth token
+    ngrok http --authtoken=$YOUR_AUTH_TOKEN 5000
 
     # run local server for debugging, pass external name from ngrok
     python manager.py runserver --external=SERVERID.ngrok.io

README.md

@@ -26,7 +26,7 @@ Campaign Configuration
 1) Create a campaign with one of several types, to determine how callers are matched to targets.
 
 * _Executive_ connects callers to the Whitehouse Switchboard, or to a specific office if known
-* _Congress_ connects callers to their Senators, Representative, or both. Uses data from the Sunlight Foundation.
+* _Congress_ connects callers to their Senators, Representative, or both. Uses the OpenStates API.
 * _State_ connects callers to their Governor or State Legislators. Uses the OpenStates API.
 * _Local_ connects callers to a local official. Campaigners must enter these numbers in advance.
 * _Custom_ can connect callers to corporate offices, local officals, or any other phone number entered in advance.
@@ -54,9 +54,9 @@ Read detailed instrustions at [INSTALLATION.md](INSTALLATION.md)
 Political Data
 --------------
 
-Political data is downloaded from Sunlight as CSV files stored in this repository. These are read on startup and saved in a memory cache for fast local lookup.
+Political data is downloaded as CSV files stored in this repository. These are read on startup and saved in a memory cache for fast local lookup.
 
-To update these files with new data after elections, run `cd call_server/political_data/data && make clean && make`, and `python manager.py load_political_data`
+To update these files with new data after elections, run `cd call_server/political_data/data && make clean && make`, and `python manager.py loadpoliticaldata`
 
 
 Code License

call_server/app.py

@@ -190,8 +190,8 @@ def inject_sitename():
         return dict(SITENAME=app.config.get('SITENAME', 'CallPower'))
 
     @app.context_processor
-    def inject_sunlight_key():
-        return dict(SUNLIGHT_API_KEY=app.config.get('SUNLIGHT_API_KEY', ''))
+    def inject_openstates_key():
+        return dict(OPENSTATES_API_KEY=app.config.get('OPENSTATES_API_KEY', ''))
 
     # json filter
     app.jinja_env.filters['json'] = json_markup

call_server/config.py

@@ -1,6 +1,5 @@
 import os
 import twilio.rest
-import sunlight
 
 
 class DefaultConfig(object):
@@ -45,10 +44,7 @@ class DefaultConfig(object):
     SECRET_KEY = os.environ.get('SECRET_KEY')
 
     GEOCODE_API_KEY = os.environ.get('GEOCODE_API_KEY')
-    SUNLIGHT_API_KEY = os.environ.get('SUNLIGHT_API_KEY')
-    if not SUNLIGHT_API_KEY:
-        SUNLIGHT_API_KEY = os.environ.get('SUNLIGHT_KEY')
-    sunlight.config.API_KEY = SUNLIGHT_API_KEY
+    OPENSTATES_API_KEY = os.environ.get('OPENSTATES_API_KEY')
 
     LOG_PHONE_NUMBERS = True
 

call_server/political_data/adapters.py

@@ -30,26 +30,23 @@ def adapt(self, data):
 class OpenStatesData(object):
     def adapt(self, data):
         mapped = {}
-        mapped['name'] = data['full_name']
-        if data['chamber'] == "upper":
-            mapped['title'] = "Senator"
-        if data['chamber'] == "lower":
-            mapped['title'] = "Representative"
-        if type(data['offices']) == list and 'phone' in data['offices'][0]:
-            mapped['number'] = data['offices'][0]['phone']
-        elif type(data['offices']) == dict and 'phone' in data['offices']:
-            mapped['number'] = data['offices']['phone']
+        mapped['name'] = data['name']
+        mapped['title'] = data['current_role']['title']
+        if type(data['offices']) == list and 'voice' in data['offices'][0]:
+            mapped['number'] = data['offices'][0]['voice']
+        elif type(data['offices']) == dict and 'voice' in data['offices']:
+            mapped['number'] = data['offices']['voice']
         else:
             mapped['number'] = None
-        mapped['uid'] = data['leg_id']
+        mapped['uid'] = data['id']
 
         return mapped
 
 
 class GovernorAdapter(object):
     def adapt(self, data):
         mapped = {}
-        mapped['name'] = u'{first_name} {last_name}'.format(**data)
+        mapped['name'] = data['name']
         mapped['title'] = data['title']
         mapped['number'] = data['phone']
         mapped['uid'] = data['state']

call_server/political_data/countries/us.py

@@ -66,18 +66,22 @@ def _load_legislators(self):
     def _load_districts(self):
         """
         Load US congressional district data from saved file
-        Returns a dictionary keyed by zipcode to cache for fast lookup
+        Returns a list of dictionaries keyed by zipcode to cache for fast lookup
 
         eg us:zipcode:94612 = [{'state':'CA', 'house_district': 13}]
         or us:zipcode:54409 = [{'state':'WI', 'house_district': 7}, {'state':'WI', 'house_district': 8}]
         """
         districts = collections.defaultdict(list)
 
         with open('call_server/political_data/data/us_districts.csv') as f:
-            reader = csv.DictReader(
-                f, fieldnames=['zipcode', 'state', 'house_district'])
+            reader = csv.DictReader(f)
 
-            for d in reader:
+            for row in reader:
+                d = {
+                    'state': row['state_abbr'],
+                    'zipcode': row['zcta'],
+                    'house_district': row['cd']
+                }
                 cache_key = self.KEY_ZIPCODE.format(**d)
                 districts[cache_key].append(d)
 

call_server/political_data/countries/us_state.py

@@ -1,8 +1,9 @@
 import csv
 import collections
+import os
 import random
 
-from sunlight import openstates, response_cache
+from requests import Session
 from . import DataProvider
 
 from ..constants import US_STATE_NAME_DICT
@@ -16,8 +17,6 @@ class USStateData(DataProvider):
 
     def __init__(self, cache, api_cache=None):
         self.cache = cache
-        if api_cache:
-            response_cache.enable(api_cache)
 
     def _load_governors(self):
         """
@@ -77,7 +76,7 @@ def locate_governor(self, state):
         return [self.KEY_GOVERNOR.format(state=state)]
 
     def locate_targets(self, latlon, chambers=TARGET_CHAMBER_BOTH, order=ORDER_IN_ORDER, state=None):
-        """ Find all state legistlators for a location, as comma delimited (lat,lon)
+        """ Find all state legislators for a location, as comma delimited (lat,lon)
             Returns a list of cached openstate keys in specified order.
         """
         if type(latlon) == tuple:
@@ -89,28 +88,36 @@ def locate_targets(self, latlon, chambers=TARGET_CHAMBER_BOTH, order=ORDER_IN_OR
             except ValueError:
                 raise ValueError('USStateData requires location as lat,lon')
 
-        legislators = openstates.legislator_geo_search(lat, lon)
+        params = dict(lat=float(lat), lng=float(lon), include='offices')
+        s = Session()
+        s.headers.update({'X-Api-Key': os.environ.get('OPENSTATES_API_KEY')})
+        response = s.get("https://v3.openstates.org/people.geo", params=params)
+        s.close()
+        if response.status_code != 200:
+            if response.status_code == 404:
+                raise NotFound("Not found: {0}".format(response.url))
+            else:
+                raise Exception(response.text)
+
+        legislators = response.json()['results']
         targets = []
         senators = []
         house_reps = []
 
-        # save full legislator data to cache
-        # just uids to result list
         for l in legislators:
-            if not l['active']:
-                # don't include inactive legislators
-                continue
-
-            if state and (state.upper() != l['state'].upper()):
+            parts = l['jurisdiction']['id'].partition('state:')
+            state_abbr = parts[2].partition("/")[0]
+            if state and (state.upper() != state_abbr.upper()):
                 # limit to one state
                 continue
 
             cache_key = self.KEY_OPENSTATES.format(**l)
             self.cache_set(cache_key, l)
 
-            if l['chamber'] == 'upper':
+            # limit to state legislators only
+            if l['current_role']['org_classification'] == 'upper' and l['jurisdiction']['classification'] == 'state':
                 senators.append(cache_key)
-            if l['chamber'] == 'lower':
+            if l['current_role']['org_classification'] == 'lower' and l['jurisdiction']['classification'] == 'state':
                 house_reps.append(cache_key)
 
         if chambers == TARGET_CHAMBER_UPPER:

call_server/political_data/data/Makefile

@@ -7,7 +7,7 @@ legislators-current.yaml :
 	curl -k "https://raw.githubusercontent.com/unitedstates/congress-legislators/master/legislators-current.yaml" -o "legislators-current.yaml"
 
 us_districts.csv:
-	curl -k "http://assets.sunlightfoundation.com/data/districts.csv" -o "us_districts.csv"
+	curl -k "https://raw.githubusercontent.com/OpenSourceActivismTech/us_zipcodes_congress/master/zccd.csv" -o "us_districts.csv"
 
 us_states.csv:
 	curl -k "https://raw.githubusercontent.com/spacedogXYZ/us_governors_contact/master/data.csv" -o "us_states.csv"