From af7bdad9f977e1d28bbde088b65b8a53cb300607 Mon Sep 17 00:00:00 2001 From: Hassan Syyid Date: Mon, 18 Nov 2024 17:41:15 -0500 Subject: [PATCH 1/2] HGI-6654: Move orders to graphql schema --- tap_shopify/schemas/orders.json | 1177 ++----------------------------- tap_shopify/streams/orders.py | 366 +++++++++- 2 files changed, 439 insertions(+), 1104 deletions(-) diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index 383494a8..6bf0e3b3 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -1,1108 +1,79 @@ { "properties": { - "presentment_currency": { - "type": [ - "null", - "string" - ] - }, - "subtotal_price_set": { "type": ["object","array", "string"] }, - "total_discounts_set": { "type": ["object", "array", "string"] }, - "total_line_items_price_set": { "type": ["object", "array", "string"] }, - "total_price_set": { "type": ["object", "array", "string"] }, - "total_shipping_price_set": { "type": ["object", "array", "string"] }, - "total_tax_set": { "type": ["object", "array", "string"] }, - "total_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "processing_method": { - "type": [ - "null", - "string" - ] - }, - "order_number": { - "type": [ - "null", - "integer" - ] - }, - "confirmed": { - "type": [ - "null", - "boolean" - ] - }, - "total_discounts": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "total_line_items_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" - }, - "shipping_lines": { - "items": { - "properties": { - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "discounted_price_set": {}, - "price_set": {}, - "price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "discount_allocations": { - "items": { - "properties": { - "discount_application_index": { - "type": [ - "null", - "integer" - ] - }, - "amount": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "discounted_price": { - "type": [ - "null", - "number" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "requested_fulfillment_service_id": { - "type": [ - "null", - "string" - ] - }, - "carrier_identifier": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "device_id": { - "type": [ - "null", - "integer" - ] - }, - "cancel_reason": { - "type": [ - "null", - "string" - ] - }, - "currency": { - "type": [ - "null", - "string" - ] - }, - "payment_gateway_names": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "source_identifier": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "processed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "referring_site": { - "type": [ - "null", - "string" - ] - }, - "contact_email": { - "type": [ - "null", - "string" - ] - }, - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "fulfillments": { - "items": { - "properties": { - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "receipt": { - "type": ["null", "object"], - "properties": { - "testcase": { - "type": ["null", "boolean"] - }, - "authorization": { - "type": ["null", "string"] - } - } - }, - "tracking_number": { - "type": [ - "null", - "string" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "shipment_status": { - "type": [ - "null", - "string" - ] - }, - "line_items": { - "$ref": "definitions.json#/line_items" - }, - "tracking_url": { - "type": [ - "null", - "string" - ] - }, - "service": { - "type": [ - "null", - "string" - ] - }, - "status": { - "type": [ - "null", - "string" - ] - }, - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "tracking_urls": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "tracking_numbers": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "tracking_company": { - "type": [ - "null", - "string" - ] - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "customer": { - "$ref": "definitions.json#/customer" - }, - "test": { - "type": [ - "null", - "boolean" - ] - }, - "total_tax": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "payment_details": { - "properties": { - "avs_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_company": { - "type": [ - "null", - "string" - ] - }, - "cvv_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_bin": { - "type": [ - "null", - "string" - ] - }, - "credit_card_number": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "source_name": { - "type": [ - "null", - "string" - ] - }, - "landing_site_ref": { - "type": [ - "null", - "string" - ] - }, - "shipping_address": { - "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "longitude": { - "type": [ - "null", - "number" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "latitude": { - "type": [ - "null", - "number" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "total_price_usd": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "closed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "discount_applications": { - "items": { - "properties": { - "target_type": { - "type": [ - "null", - "string" - ] - }, - "code": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "target_selection": { - "type": [ - "null", - "string" - ] - }, - "allocation_method": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "value_type": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "source_url": { - "type": [ - "null", - "string" - ] - }, - "subtotal_price": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "billing_address": { - "properties": { - "phone": { - "type": [ - "null", - "string" - ] - }, - "country": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "address1": { - "type": [ - "null", - "string" - ] - }, - "longitude": { - "type": [ - "null", - "number" - ] - }, - "address2": { - "type": [ - "null", - "string" - ] - }, - "last_name": { - "type": [ - "null", - "string" - ] - }, - "first_name": { - "type": [ - "null", - "string" - ] - }, - "province": { - "type": [ - "null", - "string" - ] - }, - "city": { - "type": [ - "null", - "string" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "latitude": { - "type": [ - "null", - "number" - ] - }, - "country_code": { - "type": [ - "null", - "string" - ] - }, - "province_code": { - "type": [ - "null", - "string" - ] - }, - "zip": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "landing_site": { - "type": [ - "null", - "string" - ] - }, - "taxes_included": { - "type": [ - "null", - "boolean" - ] - }, - "token": { - "type": [ - "null", - "string" - ] - }, - "app_id": { - "type": [ - "null", - "integer" - ] - }, - "total_tip_received": { - "type": [ - "null", - "string" - ] - }, - "browser_ip": { - "type": [ - "null", - "string" - ] - }, - "discount_codes": { - "items": { - "properties": { - "code": { - "type": [ - "null", - "string" - ] - }, - "amount": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, - "type": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "tax_lines": { - "$ref": "definitions.json#/tax_lines" - }, - "phone": { - "type": [ - "null", - "string" - ] - }, - "note_attributes": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "fulfillment_status": { - "type": [ - "null", - "string" - ] - }, - "order_status_url": { - "type": [ - "null", - "string" - ] - }, - "client_details": { - "properties": { - "session_hash": { - "type": [ - "null", - "string" - ] - }, - "accept_language": { - "type": [ - "null", - "string" - ] - }, - "browser_width": { - "type": [ - "null", - "integer" - ] - }, - "user_agent": { - "type": [ - "null", - "string" - ] - }, - "browser_ip": { - "type": [ - "null", - "string" - ] - }, - "browser_height": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "buyer_accepts_marketing": { - "type": [ - "null", - "boolean" - ] - }, - "checkout_token": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "string" - ] - }, - "financial_status": { - "type": [ - "null", - "string" - ] - }, - "customer_locale": { - "type": [ - "null", - "string" - ] - }, - "checkout_id": { - "type": [ - "null", - "integer" - ] - }, - "total_weight": { - "type": [ - "null", - "integer" - ] - }, - "gateway": { - "type": [ - "null", - "string" - ] - }, - "cart_token": { - "type": [ - "null", - "string" - ] - }, - "cancelled_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "refunds": { - "items": { - "properties": { - "admin_graphql_api_id": { - "type": [ - "null", - "string" - ] - }, - "refund_line_items": { - "items": { - "properties": { - "line_item": { - "$ref": "definitions.json#/line_item" - }, - "location_id": { - "type": [ - "null", - "integer" - ] - }, - "line_item_id": { - "type": [ - "null", - "integer" - ] - }, - "quantity": { - "type": [ - "null", - "integer" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "total_tax": { - "type": [ - "null", - "number" - ] - }, - "restock_type": { - "type": [ - "null", - "string" - ] - }, - "subtotal": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "restock": { - "type": [ - "null", - "boolean" - ] - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "user_id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "processed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "order_adjustments": { - "$ref": "definitions.json#/order_adjustments" - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "reference": { - "type": [ - "null", - "string" - ] + "id": { "type": ["null", "string"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, + "presentmentCurrencyCode": { "type": ["null", "string"] }, + "subtotalPriceSet": { "type": ["null", "object"] }, + "totalDiscountsSet": { "type": ["null", "object"] }, + "totalPriceSet": { "type": ["null", "object"] }, + "totalShippingPriceSet": { "type": ["null", "object"] }, + "totalTaxSet": { "type": ["null", "object"] }, + "test": { "type": ["null", "boolean"] }, + "app": { "type": ["null", "object"] }, + "billingAddress": { "type": ["null", "object"] }, + "cancelReason": { "type": ["null", "string"] }, + "cancelledAt": { "type": ["null", "string"], "format": "date-time" }, + "clientIp": { "type": ["null", "string"] }, + "closedAt": { "type": ["null", "string"], "format": "date-time" }, + "confirmed": { "type": ["null", "boolean"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "currencyCode": { "type": ["null", "string"] }, + "customAttributes": { + "type": ["null", "array"], + "items": {} + }, + "customer": { "type": ["null", "object"] }, + "customerAcceptsMarketing": { "type": ["null", "boolean"] }, + "customerLocale": { "type": ["null", "string"] }, + "discountCodes": { + "type": ["null", "array"], + "items": {} + }, + "displayFinancialStatus": { "type": ["null", "string"] }, + "displayFulfillmentStatus": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "landingPageUrl": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "note": { "type": ["null", "string"] }, + "paymentGatewayNames": { + "type": ["null", "array"], + "items": {} + }, + "phone": { "type": ["null", "string"] }, + "processedAt": { "type": ["null", "string"], "format": "date-time" }, + "referrerUrl": { "type": ["null", "string"] }, + "registeredSourceUrl": { "type": ["null", "string"] }, + "shippingAddress": { "type": ["null", "object"] }, + "sourceIdentifier": { "type": ["null", "string"] }, + "tags": { + "type": ["null", "array"], + "items": {} + }, + "totalWeight": { "type": ["null", "integer"] }, + "totalTipReceivedSet": { "type": ["null", "object"] }, + "taxLines": { + "type": ["null", "array"], + "items": {} + }, + "refunds": { + "type": ["null", "array"], + "items": {} + }, + "discountApplications": { + "type": ["null", "array"], + "items": {} + }, + "fulfillments": { + "type": ["null", "array"], + "items": {} + }, + "shippingLines": { + "type": ["null", "array"], + "items": {} + }, + "lineItems": { + "type": ["null", "array"], + "items": {} } }, "type": "object" diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index bb360204..2ea40ae2 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1,10 +1,374 @@ +import os +import sys +import json +import time + +import singer import shopify from tap_shopify.context import Context -from tap_shopify.streams.base import Stream +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + +class HiddenPrints: + def __enter__(self): + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.stdout.close() + sys.stdout = self._original_stdout + +def unwrap_nodes(obj): + """ + Recursively unwraps `nodes` keys in a nested dictionary or list. + """ + if isinstance(obj, dict): + # Check if the dictionary has a single key 'nodes' + if len(obj) == 1 and "nodes" in obj: + return unwrap_nodes(obj["nodes"]) + else: + # Recursively process each value in the dictionary + return {k: unwrap_nodes(v) for k, v in obj.items()} + elif isinstance(obj, list): + # Recursively process each item in the list + return [unwrap_nodes(item) for item in obj] + else: + # If it's neither a dict nor a list, return the object as-is + return obj class Orders(Stream): name = 'orders' replication_object = shopify.Order + gql_query = """ + query Orders($query: String, $cursor: String) { + orders(first: 250, query: $query, after: $cursor) { + nodes { + id + updatedAt + presentmentCurrencyCode + subtotalPriceSet { + shopMoney { + amount + currencyCode + } + } + totalDiscountsSet { + shopMoney { + amount + currencyCode + } + } + totalPriceSet { + shopMoney { + amount + currencyCode + } + } + totalShippingPriceSet { + shopMoney { + amount + currencyCode + } + } + totalTaxSet { + shopMoney { + amount + currencyCode + } + } + test + app { + id + } + billingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCode + countryCodeV2 + firstName + formatted + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + zip + } + cancelReason + cancelledAt + clientIp + closedAt + confirmed + createdAt + currencyCode + customAttributes { + key + value + } + customer { + canDelete + createdAt + displayName + email + firstName + hasTimelineComment + id + lastName + legacyResourceId + lifetimeDuration + locale + multipassIdentifier + note + numberOfOrders + phone + productSubscriberStatus + state + tags + taxExempt + taxExemptions + unsubscribeUrl + updatedAt + validEmailAddress + verifiedEmail + } + customerAcceptsMarketing + customerLocale + discountCodes + displayFinancialStatus + displayFulfillmentStatus + email + landingPageUrl + name + note + paymentGatewayNames + phone + processedAt + referrerUrl + registeredSourceUrl + shippingAddress { + address1 + address2 + city + company + coordinatesValidated + country + countryCode + countryCodeV2 + firstName + formatted + formattedArea + id + lastName + latitude + longitude + name + phone + province + provinceCode + timeZone + zip + } + sourceIdentifier + tags + totalWeight + totalTipReceivedSet { + shopMoney { + amount + currencyCode + } + } + taxLines { + channelLiable + price + rate + title + } + refunds(first: 250) { + createdAt + id + legacyResourceId + note + updatedAt + refundLineItems(first: 250) { + nodes { + price + quantity + restockType + restocked + subtotal + totalTax + lineItem { + canRestock + currentQuantity + discountedTotal + discountedUnitPrice + fulfillableQuantity + fulfillmentStatus + id + merchantEditable + name + nonFulfillableQuantity + originalTotal + originalUnitPrice + quantity + refundableQuantity + requiresShipping + restockable + sku + taxable + title + totalDiscount + unfulfilledDiscountedTotal + unfulfilledOriginalTotal + unfulfilledQuantity + variantTitle + vendor + } + } + } + } + discountApplications(first: 250) { + nodes { + allocationMethod + index + targetSelection + targetType + value { + ... on MoneyV2 { + amount + currencyCode + } + } + } + } + fulfillments { + createdAt + deliveredAt + displayStatus + estimatedDeliveryAt + id + inTransitAt + legacyResourceId + name + requiresShipping + status + totalQuantity + updatedAt + } + shippingLines(first: 250) { + nodes { + carrierIdentifier + code + custom + deliveryCategory + id + phone + price + shippingRateHandle + source + title + } + } + lineItems(first: 250) { + nodes { + canRestock + currentQuantity + discountedTotal + discountedUnitPrice + fulfillableQuantity + fulfillmentStatus + id + merchantEditable + name + nonFulfillableQuantity + originalTotal + originalUnitPrice + quantity + refundableQuantity + requiresShipping + restockable + sku + taxable + title + totalDiscount + unfulfilledDiscountedTotal + unfulfilledOriginalTotal + unfulfilledQuantity + variantTitle + vendor + } + } + } + pageInfo { + endCursor + hasNextPage + hasPreviousPage + startCursor + } + } + } + """ + + @shopify_error_handling + def call_api_for_orders(self, gql_client, query, cursor=None): + with HiddenPrints(): + response = gql_client.execute(self.gql_query, dict(query=query, cursor=cursor)) + result = json.loads(response) + if result.get("errors"): + raise Exception(result['errors']) + return result + + def get_orders(self, query): + gql_client = shopify.GraphQL() + page = self.call_api_for_orders(gql_client, query) + yield page + + # paginate + page_info = page['data']['orders']['pageInfo'] + while page_info['hasNextPage']: + page = self.call_api_for_orders(gql_client, query, cursor=page_info['endCursor']) + page_info = page['data']['orders']['pageInfo'] + yield page + + def get_objects(self): + # get bookmark + updated_at = self.get_bookmark().isoformat() + query = f"updated_at:>'{updated_at}'" + orders = 0 + start = time.time() + + for page in self.get_orders(query): + for order in page['data']['orders']['nodes']: + # TODO: need to map back to Shopify REST formatting + orders += 1 + now = time.time() + LOGGER.info(f"Got {orders} in {now - start} sec.") + + # unwrap nodes + order = unwrap_nodes(order) + + yield order + + def sync(self): + """Yield's processed SDK object dicts to the caller. + + This is the default implementation. Get's all of self's objects + and calls to_dict on them with no further processing. + """ + for obj in self.get_objects(): + yield obj + Context.stream_objects['orders'] = Orders From ef434307952cb111753de3933b75a1fb205d9c6d Mon Sep 17 00:00:00 2001 From: Hassan Syyid Date: Mon, 18 Nov 2024 18:06:23 -0500 Subject: [PATCH 2/2] Remove debug performance info --- tap_shopify/streams/orders.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tap_shopify/streams/orders.py b/tap_shopify/streams/orders.py index 2ea40ae2..cba1ea03 100644 --- a/tap_shopify/streams/orders.py +++ b/tap_shopify/streams/orders.py @@ -1,7 +1,6 @@ import os import sys import json -import time import singer import shopify @@ -347,15 +346,10 @@ def get_objects(self): # get bookmark updated_at = self.get_bookmark().isoformat() query = f"updated_at:>'{updated_at}'" - orders = 0 - start = time.time() for page in self.get_orders(query): for order in page['data']['orders']['nodes']: # TODO: need to map back to Shopify REST formatting - orders += 1 - now = time.time() - LOGGER.info(f"Got {orders} in {now - start} sec.") # unwrap nodes order = unwrap_nodes(order)