HGI-6654/ Improve request time and increase default results_per_page

tap_shopify/__init__.py

@@ -33,6 +33,7 @@ def initialize_shopify_client():
     version = Context.config.get('api_version', '2024-01')
     session = shopify.Session(shop, version, api_key)
     shopify.ShopifyResource.activate_session(session)
+    shopify.ShopifyResource._headers['Connection'] = 'keep-alive'
     # Shop.current() makes a call for shop details with provided shop and api_key
     return shopify.Shop.current().attributes
 
@@ -183,7 +184,7 @@ def sync():
 
         Context.state['bookmarks'].pop('currently_sync_stream')
         singer.write_state(Context.state)
-
+    
     LOGGER.info('----------------------')
     for stream_id, stream_count in Context.counts.items():
         LOGGER.info('%s: %d', stream_id, stream_count)
@@ -226,6 +227,9 @@ def main():
             raise ShopifyError(exc, msg) from exc
     except Exception as exc:
         raise ShopifyError(exc) from exc
+    finally:
+        shopify.ShopifyResource.clear_session()
+
 
 if __name__ == "__main__":
     main()

tap_shopify/context.py

@@ -3,6 +3,8 @@
 
 LOGGER = singer.get_logger()
 
+RESULTS_PER_PAGE = 250
+
 class Context():
     config = {}
     state = {}
@@ -27,7 +29,7 @@ def is_selected(cls, stream_name):
     def get_results_per_page(cls, default_results_per_page):
         results_per_page = default_results_per_page
         try:
-            results_per_page = int(cls.config.get("results_per_page"))
+            results_per_page = int(cls.config.get("results_per_page", RESULTS_PER_PAGE))
         except TypeError:
             # None value or no key
             pass

tap_shopify/streams/base.py

@@ -9,17 +9,15 @@
 import simplejson
 import singer
 from singer import metrics, utils
-from tap_shopify.context import Context
+from tap_shopify.context import Context, RESULTS_PER_PAGE
 
 LOGGER = singer.get_logger()
 
-RESULTS_PER_PAGE = 175
-
 # We've observed 500 errors returned if this is too large (30 days was too
 # large for a customer)
 DATE_WINDOW_SIZE = 365
 
-# We will retry a 500 error a maximum of 5 times before giving up
+# We will retry a 500 error a maximum of 10 times before giving up
 MAX_RETRIES = 10
 
 def is_not_status_code_fn(status_code):

tap_shopify/streams/inventory_items.py

@@ -1,6 +1,5 @@
 import singer
 import shopify
-from singer.utils import strftime,strptime_to_utc
 from tap_shopify.streams.base import (Stream, shopify_error_handling)
 from tap_shopify.context import Context