Merge pull request #56 from useshortcut/daniel/sc-264542/throttle-ent…

…ity-creation-rate

Limit Shortcut API requests to respect rate limits

pivotal-import/Pipfile

@@ -5,6 +5,7 @@ name = "pypi"
 
 [packages]
 requests = "~=2.31.0"
+pyrate-limiter = "~=3.6.0"
 
 [dev-packages]
 black = "*"

pivotal-import/README.md

@@ -8,7 +8,7 @@ In order to run this, you will require a Pivotal account and the ability to sign
 
 1. Sign up for a Shortcut account at [https://www.shortcut.com/signup](https://www.shortcut.com/signup).
    - **NOTE:** Do not run this importer against an existing Shortcut workspace that already has data you wish to keep.
-1. [Create an API token](https://app.shortcut.com/settings/account/api-tokens) and [export it into your environment](../Authentication.md).
+1. [Create a new API token](https://app.shortcut.com/settings/account/api-tokens) and [export it into your environment](../Authentication.md). Ensure you use this token only for this importer, so that you aren't rate limited unexpectedly by the Shortcut API.
 1. Export your Pivotal project to CSV and save the file to `data/pivotal_export.csv`.
 1. Create/Invite all users you want to reference into your Shortcut workspace.
    - **NOTE:** If you're not on a Shortcut trial, please [reach out to our support team](https://help.shortcut.com/hc/en-us/requests/new) before running this import to make sure you're not billed for users that you want to be disabled after import.

pivotal-import/delete_imported_entities.py

@@ -27,8 +27,9 @@ def delete_entity(entity_type, entity_id):
     if prefix:
         try:
             sc_delete(f"{prefix}{entity_id}")
-        except requests.HTTPError:
+        except requests.HTTPError as err:
             printerr(f"Unable to delete {entity_type} {entity_id}")
+            printerr(f"Error: {err}")
             return None
 
     return True
@@ -39,6 +40,8 @@ def main(argv):
     if args.debug:
         logging.basicConfig(level=logging.DEBUG)
 
+    print_rate_limiting_explanation()
+
     counter = Counter()
     with open(shortcut_imported_entities_csv) as csvfile:
         reader = csv.DictReader(csvfile)
@@ -49,6 +52,11 @@ def main(argv):
             if args.apply:
                 if delete_entity(entity_type, entity_id):
                     counter[entity_type] += 1
+                    # Enhancement: This can be replaced with a link to a relevant label,
+                    # which is done during import because there is a trivially simple
+                    # place in the import code flow to print the Shortcut-provided
+                    # app_url for the import-specific label.
+                    print("Deleted {} {}".format(entity_type, entity_id))
             else:
                 counter[entity_type] += 1
 

pivotal-import/lib.py

@@ -14,11 +14,43 @@
 import os
 import logging
 
+from pyrate_limiter import Duration, InMemoryBucket, Limiter, Rate
 import requests
 
 # Logging
 logger = logging.getLogger(__name__)
 
+# Rate limiting. See https://developer.shortcut.com/api/rest/v3#Rate-Limiting
+# The Shortcut API limit is 200 per minute; the 200th request within 60 seconds
+# will receive an HTTP 429 response.
+#
+# The rate limiting config below sets an in-memory limit that is just below
+# Shortcut's rate limit to reduce the possibility of being throttled, and sets
+# the amount of time it will wait once it reaches that limit to just
+# over a minute to account for possible computer clock differences.
+max_requests_per_minute = 200
+rate = Rate(max_requests_per_minute - 5, Duration.MINUTE)
+bucket = InMemoryBucket([rate])
+max_limiter_delay_seconds = 70
+limiter = Limiter(
+    bucket, raise_when_fail=True, max_delay=Duration.SECOND * max_limiter_delay_seconds
+)
+
+
+def rate_mapping(*args, **kwargs):
+    return "shortcut-api-request", 1
+
+
+rate_decorator = limiter.as_decorator()
+
+
+def print_rate_limiting_explanation():
+    printerr(
+        f"""[Note] This importer adheres to the Shortcut API rate limit of {max_requests_per_minute} requests per minute.
+       It may pause for up to {max_limiter_delay_seconds} seconds during processing to avoid request throttling."""
+    )
+
+
 # API Helpers
 sc_token = os.getenv("SHORTCUT_API_TOKEN")
 api_url_base = "https://api.app.shortcut.com/api/v3"
@@ -30,6 +62,7 @@
 }
 
 
+@rate_decorator(rate_mapping)
 def sc_get(path, params={}):
     """
     Make a GET api call.
@@ -43,6 +76,7 @@ def sc_get(path, params={}):
     return resp.json()
 
 
+@rate_decorator(rate_mapping)
 def sc_post(path, data={}):
     """Make a POST api call.
 
@@ -59,6 +93,7 @@ def sc_post(path, data={}):
     return resp.json()
 
 
+@rate_decorator(rate_mapping)
 def sc_put(path, data={}):
     """
     Make a PUT api call.
@@ -73,6 +108,7 @@ def sc_put(path, data={}):
     return resp.json()
 
 
+@rate_decorator(rate_mapping)
 def sc_delete(path):
     """
     Make a DELETE api call.

pivotal-import/pivotal_import.py

@@ -509,6 +509,7 @@ def main(argv):
 
     cfg = load_config()
     ctx = build_ctx(cfg)
+    print_rate_limiting_explanation()
     process_pt_csv_export(ctx, cfg["pt_csv_file"], entity_collector)
 
     created_entities = entity_collector.commit()