Skip to content

Commit

Permalink
feat: add DATA WAREHOUSE to integrations object in shopify pixel sour…
Browse files Browse the repository at this point in the history
…ce (#3980)

* feat: add redis support in shopify pixel for id stitching

* chore: update redis set structure

* chore: update attribute name

Co-authored-by: Utsab Chowdhury <[email protected]>

* fix: stitched events through redis in pixel

* fix: stitched events through redis in pixel

* fix: stitched events through redis in pixel

* chore: add tests

* chore: add campaign object support in web pixel events (#3973)

* chore: add campaign object support in web pixel events

* chore: add custom utm support

* chore: address comments

* feat: add DATA WAREHOUSE to intg object in shopify pixel

* feat: add DATA WAREHOUSE json path handling for some providers

* chore: update json structure to be event specific in shopify

* chore: add test

* chore: add testx2

* chore: resolve conflict

* chore: address comments, refactor wh changes

* chore: refactor

* chore: resolve conflicts

* chore: fix test

* feat: add redis support in shopify pixel for id stitching (#3957)

* feat: add redis support in shopify pixel for id stitching (#3957)

* chore: fix redis test

* chore: pixelEventLabel for identifier events also

* fix: redis await missing

* fix: setProperty issue

* fix: refactor basic code flow

* chore: handle imports, await functions

* chore: add pixel prefix for redis carttoken in shopify

* chore: address comments

* chore: add stat for serverside missing cart token

* chore: add unit test for utm param extractfunction

* chore: fix sonar issues in wh index js

---------

Co-authored-by: Utsab Chowdhury <[email protected]>
  • Loading branch information
yashasvibajpai and utsabc authored Jan 23, 2025
1 parent 23ad10a commit 3c20393
Show file tree
Hide file tree
Showing 11 changed files with 243 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -123,14 +123,19 @@ const handleCommonProperties = (message, event, shopifyTopic) => {
if (shopifyTopic === 'orders_updated') {
message.setProperty(`context.order_token`, event.token);
}
message.setProperty('integrations.DATA_WAREHOUSE', {
options: {
jsonPaths: [`${message.type}.context.shopifyDetails`],
},
});
return message;
};

module.exports = {
createPropertiesForEcomEventFromWebhook,
getCartToken,
getProductsFromLineItems,
getAnonymousIdFromAttributes,
getCartToken,
setAnonymousId,
handleCommonProperties,
};
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,11 @@ function processPixelEvent(inputEvent) {
}
message.anonymousId = clientId;
message.setProperty(`integrations.${INTEGERATION}`, true);
message.setProperty('integrations.DATA_WAREHOUSE', {
options: {
jsonPaths: [`${message.type}.context.shopifyDetails`],
},
});
message.setProperty('context.library', {
name: 'RudderStack Shopify Cloud',
eventOrigin: 'client',
Expand Down
11 changes: 7 additions & 4 deletions src/warehouse/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const {
validTimestamp,
getVersionedUtils,
isRudderSourcesEvent,
mergeJSONPathsFromDataWarehouse,
} = require('./util');
const { getMergeRuleEvent } = require('./identity');

Expand Down Expand Up @@ -307,8 +308,8 @@ function isStringLikeObject(obj) {
let minKey = Infinity;
let maxKey = -Infinity;

for (let i = 0; i < keys.length; i++) {
const key = keys[i];
for (const element of keys) {
const key = element;
const value = obj[key];

if (!isNonNegativeInteger(key)) return false;
Expand Down Expand Up @@ -336,8 +337,8 @@ function stringLikeObjectToString(obj) {
.sort((a, b) => a - b);
let result = '';

for (let i = 0; i < keys.length; i++) {
result += obj[keys[i].toString()];
for (const element of keys) {
result += obj[element.toString()];
}

return result;
Expand Down Expand Up @@ -655,6 +656,8 @@ function processWarehouseMessage(message, options) {
const skipReservedKeywordsEscaping =
options.integrationOptions.skipReservedKeywordsEscaping || false;

mergeJSONPathsFromDataWarehouse(message, options);

// underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_"
// underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that
// For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false.
Expand Down
22 changes: 22 additions & 0 deletions src/warehouse/util.js
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,27 @@ const getRecordIDForExtract = (message) => {
return recordId;
};

function mergeJSONPathsFromDataWarehouse(message, options) {
const dataWarehouseOptions = message.integrations?.['DATA_WAREHOUSE']?.options;
if (!dataWarehouseOptions?.jsonPaths) return;

const dataWarehouseJSONPaths = Array.isArray(dataWarehouseOptions.jsonPaths)
? dataWarehouseOptions.jsonPaths
: [];
const currentJSONPaths = Array.isArray(options.integrationOptions?.jsonPaths)
? options.integrationOptions.jsonPaths
: [];

switch (options.provider) {
case 'rs':
case 'postgres':
case 'snowflake':
case 'bq':
options.integrationOptions.jsonPaths = [...dataWarehouseJSONPaths, ...currentJSONPaths];
break;
}
}

module.exports = {
isObject,
isValidJsonPathKey,
Expand All @@ -148,4 +169,5 @@ module.exports = {
sourceCategoriesToUseRecordId,
getCloudRecordID,
getRecordIDForExtract,
mergeJSONPathsFromDataWarehouse,
};
103 changes: 103 additions & 0 deletions test/__tests__/shopify_warehouse.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
const event = {
"request": {
"query": {
"whSchemaVersion": "v1"
}
},
"message": {
"context": {
"shopifyDetails": {
"id": 5778367414385,
"current_total_tax": "10.00",
"current_total_tax_set": {
"shop_money": {
"amount": "10.00",
"currency_code": "USD"
},
},
"name": "#1017",
"phone": null,
}
},
"integrations": {
"SHOPIFY": true,
"DATA_WAREHOUSE": {
"options": {
"jsonPaths": [
"track.context.shopifyDetails"
]
}
}
},
"type": "track",
"event": "Order Updated",
"properties": {
"order_id": "5778367414385",
"currency": "USD",
"products": [
{
"product_id": "7234590408817",
"price": 600,
"quantity": 1
}
]
},
"userId": "123321",
"traits": {},
"timestamp": "2024-01-01T01:23:45.678Z",
},
"destination": {
"Config": {},
}
};

/*
Test for warehouse agnostic DATA_WAREHOUSE JSON column support for Shopify source
*/
describe('DATA_WAREHOUSE integrations', () => {
it('should process event and return responses for common providers for agnostic support', () => {
const responses = require('../../src/v0/destinations/snowflake/transform').process(event);
expect(responses).toHaveLength(2);
expect(responses[0].metadata.table).toBe('TRACKS');
expect(responses[1].metadata.table).toBe('ORDER_UPDATED');

expect(responses[0].metadata.columns.CONTEXT_SHOPIFY_DETAILS).toBe('json');
expect(responses[0].data.CONTEXT_SHOPIFY_DETAILS).toBe('{"id":5778367414385,"current_total_tax":"10.00","current_total_tax_set":{"shop_money":{"amount":"10.00","currency_code":"USD"}},"name":"#1017","phone":null}');

expect(responses[1].metadata.columns.CONTEXT_SHOPIFY_DETAILS).toBe('json');
expect(responses[1].data.CONTEXT_SHOPIFY_DETAILS).toBe('{"id":5778367414385,"current_total_tax":"10.00","current_total_tax_set":{"shop_money":{"amount":"10.00","currency_code":"USD"}},"name":"#1017","phone":null}');
});

it('should process event and return response for other providers like mssql', () => {
const responses = require('../../src/v0/destinations/mssql/transform').process(event);
expect(responses).toHaveLength(2);
expect(responses[0].metadata.table).toBe('tracks');
expect(responses[1].metadata.table).toBe('order_updated');

expect(responses[0].metadata.columns.context_shopify_details).toBe(undefined);
expect(responses[0].metadata.columns.context_shopify_details_id).toBe('int');
expect(responses[0].metadata.columns.context_shopify_details_current_total_tax).toBe('string');
expect(responses[0].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('string');
expect(responses[0].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('string');
expect(responses[0].metadata.columns.context_shopify_details_name).toBe('string');
expect(responses[0].data.context_shopify_details).toBe(undefined);
expect(responses[0].data.context_shopify_details_id).toBe(5778367414385);
expect(responses[0].data.context_shopify_details_current_total_tax).toBe('10.00');
expect(responses[0].data.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('10.00');
expect(responses[0].data.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('USD');
expect(responses[0].data.context_shopify_details_name).toBe('#1017');

expect(responses[1].metadata.columns.context_shopify_details).toBe(undefined);
expect(responses[1].metadata.columns.context_shopify_details_id).toBe('int');
expect(responses[1].metadata.columns.context_shopify_details_current_total_tax).toBe('string');
expect(responses[1].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('string');
expect(responses[1].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('string');
expect(responses[1].metadata.columns.context_shopify_details_name).toBe('string');
expect(responses[1].data.context_shopify_details).toBe(undefined);
expect(responses[1].data.context_shopify_details_id).toBe(5778367414385);
expect(responses[1].data.context_shopify_details_current_total_tax).toBe('10.00');
expect(responses[1].data.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('10.00');
expect(responses[1].data.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('USD');
expect(responses[1].data.context_shopify_details_name).toBe('#1017');
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,11 @@ export const pixelCheckoutEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Checkout Started',
Expand Down Expand Up @@ -775,6 +780,11 @@ export const pixelCheckoutEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Order Completed',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,11 @@ export const pixelCheckoutStepsScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Checkout Address Info Submitted',
Expand Down Expand Up @@ -921,6 +926,11 @@ export const pixelCheckoutStepsScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Checkout Contact Info Submitted',
Expand Down Expand Up @@ -1470,6 +1480,11 @@ export const pixelCheckoutStepsScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Checkout Shipping Info Submitted',
Expand Down Expand Up @@ -2035,6 +2050,11 @@ export const pixelCheckoutStepsScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Payment Info Entered',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['page.context.shopifyDetails'],
},
},
},
name: 'Page View',
type: 'page',
Expand Down Expand Up @@ -179,6 +184,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Product Viewed',
Expand Down Expand Up @@ -343,6 +353,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Cart Viewed',
Expand Down Expand Up @@ -568,6 +583,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Product List Viewed',
Expand Down Expand Up @@ -738,6 +758,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Product Added',
Expand Down Expand Up @@ -879,6 +904,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Product Removed',
Expand Down Expand Up @@ -968,6 +998,11 @@ export const pixelEventsTestScenarios = [
},
integrations: {
SHOPIFY: true,
DATA_WAREHOUSE: {
options: {
jsonPaths: ['track.context.shopifyDetails'],
},
},
},
type: 'track',
event: 'Search Submitted',
Expand Down
Loading

0 comments on commit 3c20393

Please sign in to comment.