diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index fcf1fe39..163b9abe 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -33,6 +33,7 @@ def initialize_shopify_client(): version = Context.config.get('api_version', '2024-01') session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) + shopify.ShopifyResource._headers['Connection'] = 'keep-alive' # Shop.current() makes a call for shop details with provided shop and api_key return shopify.Shop.current().attributes @@ -183,7 +184,7 @@ def sync(): Context.state['bookmarks'].pop('currently_sync_stream') singer.write_state(Context.state) - + LOGGER.info('----------------------') for stream_id, stream_count in Context.counts.items(): LOGGER.info('%s: %d', stream_id, stream_count) @@ -226,6 +227,9 @@ def main(): raise ShopifyError(exc, msg) from exc except Exception as exc: raise ShopifyError(exc) from exc + finally: + shopify.ShopifyResource.clear_session() + if __name__ == "__main__": main() diff --git a/tap_shopify/context.py b/tap_shopify/context.py index 3408bb8f..6200ab0e 100644 --- a/tap_shopify/context.py +++ b/tap_shopify/context.py @@ -3,6 +3,8 @@ LOGGER = singer.get_logger() +RESULTS_PER_PAGE = 250 + class Context(): config = {} state = {} @@ -27,7 +29,7 @@ def is_selected(cls, stream_name): def get_results_per_page(cls, default_results_per_page): results_per_page = default_results_per_page try: - results_per_page = int(cls.config.get("results_per_page")) + results_per_page = int(cls.config.get("results_per_page", RESULTS_PER_PAGE)) except TypeError: # None value or no key pass diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index b198b3c3..a1d0a37e 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -9,17 +9,15 @@ import simplejson import singer from singer import metrics, utils -from tap_shopify.context import Context +from tap_shopify.context import Context, RESULTS_PER_PAGE LOGGER = singer.get_logger() -RESULTS_PER_PAGE = 175 - # We've observed 500 errors returned if this is too large (30 days was too # large for a customer) DATE_WINDOW_SIZE = 365 -# We will retry a 500 error a maximum of 5 times before giving up +# We will retry a 500 error a maximum of 10 times before giving up MAX_RETRIES = 10 def is_not_status_code_fn(status_code): diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 89088513..9f545a3b 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -1,6 +1,5 @@ import singer import shopify -from singer.utils import strftime,strptime_to_utc from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context