Merge pull request #5 from hotgluexyz/fix/retries

improve and mute backoff

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="tap-shopify",
-    version="1.4.5",
+    version="1.4.6",
     description="Singer.io tap for extracting Shopify data",
     author="Stitch",
     url="http://github.com/singer-io/tap-shopify",

tap_shopify/__init__.py

@@ -5,6 +5,7 @@
 import time
 import math
 import copy
+import logging
 
 import pyactiveresource
 import shopify
@@ -14,12 +15,16 @@
 from singer import Transformer
 from tap_shopify.context import Context
 from tap_shopify.exceptions import ShopifyError
+from tap_shopify.streams.base import shopify_error_handling
 import tap_shopify.streams # Load stream objects into Context
 
 REQUIRED_CONFIG_KEYS = ["shop"]
 LOGGER = singer.get_logger()
 SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'}
 
+logging.getLogger('backoff').setLevel(logging.CRITICAL)
+
+@shopify_error_handling
 def initialize_shopify_client():
     api_key = Context.config.get('access_token', Context.config.get("api_key"))
     shop = Context.config['shop']

tap_shopify/streams/locations.py

@@ -8,8 +8,11 @@ class Locations(Stream):
     replication_object = shopify.Location
 
     @shopify_error_handling
+    def api_call_for_locations_data(self):
+        return self.replication_object.find()
+
     def get_locations_data(self):
-        location_page = self.replication_object.find()
+        location_page = self.api_call_for_locations_data()
         yield from location_page
 
         while location_page.has_next_page():

tap_shopify/streams/shop.py

@@ -1,15 +1,36 @@
 import shopify
-
-from tap_shopify.streams.base import Stream
+from singer import utils
+from tap_shopify.streams.base import (Stream, shopify_error_handling)
 from tap_shopify.context import Context
 
-
 class Shop(Stream):
     name = 'shop'
     replication_object = shopify.Shop
 
+    @shopify_error_handling
+    def api_call_for_shop_data(self):
+        return self.replication_object.current()
+
+    def get_shop_data(self):
+        shop_page = [self.api_call_for_shop_data()]
+        yield from shop_page
+
     def sync(self):
-        response = shopify.Shop.current()
-        return [response.to_dict()]
+        bookmark = self.get_bookmark()
+        max_bookmark = bookmark
+
+        for shop in self.get_shop_data():
+
+            shop_dict = shop.to_dict()
+            replication_value = utils.strptime_to_utc(shop_dict[self.replication_key])
+
+            if replication_value >= bookmark:
+                yield shop_dict
+
+            # update max bookmark if "replication_value" of current shop is greater
+            if replication_value > max_bookmark:
+                max_bookmark = replication_value
+
+        self.update_bookmark(utils.strftime(max_bookmark))
 
-Context.stream_objects['shop'] = Shop
+Context.stream_objects['shop'] = Shop