From ef9356afa28dba7801df1b2abe098f2ecec522dd Mon Sep 17 00:00:00 2001 From: mihir-4116 Date: Thu, 30 Nov 2023 10:44:33 +0530 Subject: [PATCH 1/3] fix(integrations): onboard sprig destination --- .../v2/destinations/sprig/procWorkflow.yaml | 71 +++ .../destinations/sprig/processor/data.ts | 504 ++++++++++++++++++ 2 files changed, 575 insertions(+) create mode 100644 src/cdk/v2/destinations/sprig/procWorkflow.yaml create mode 100644 test/integrations/destinations/sprig/processor/data.ts diff --git a/src/cdk/v2/destinations/sprig/procWorkflow.yaml b/src/cdk/v2/destinations/sprig/procWorkflow.yaml new file mode 100644 index 00000000000..3ad0d7f841f --- /dev/null +++ b/src/cdk/v2/destinations/sprig/procWorkflow.yaml @@ -0,0 +1,71 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: defaultRequestConfig + path: ../../../../v0/util + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + + - name: validateInput + template: | + let messageType = $.outputs.messageType; + $.assert(messageType, "message Type is not present. Aborting"); + $.assert(messageType in {{$.EventType.([.IDENTIFY, .TRACK])}}, "message type " + messageType + " is not supported"); + $.assertConfig(.destination.Config.apiKey, "API Key is not present. Aborting"); + + - name: prepareIdentifyPayload + condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} + template: | + $.context.payload = .message.({ + userId: {{{{$.getGenericPaths("userIdOnly")}}}}, + emailAddress: {{{{$.getGenericPaths("email")}}}}, + attributes: .context.traits + }); + $.context.payload = $.removeUndefinedAndNullValues($.context.payload); + + - name: validateIdentifyPayload + condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} + template: | + $.assert($.context.payload.userId, "userId is required"); + + - name: prepareTrackPayload + condition: $.outputs.messageType === {{$.EventType.TRACK}} + template: | + $.context.payload = .message.({ + userId: {{{{$.getGenericPaths("userIdOnly")}}}}, + emailAddress: {{{{$.getGenericPaths("email")}}}} + }); + const events = [ + { + event: .message.event, + timestamp: $.toMilliseconds(.message.().({{{{$.getGenericPaths("timestamp")}}}})) + } + ] + $.context.payload.events = events; + $.context.payload = $.removeUndefinedAndNullValues($.context.payload); + + - name: validateTrackPayload + condition: $.outputs.messageType === {{$.EventType.TRACK}} + template: | + $.assert($.context.payload.userId, "userId is required"); + $.assert($.context.payload.events[0].event, "event name is required"); + + - name: buildResponse + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = $.context.payload; + response.endpoint = "https://api.sprig.com/v2/users"; + response.headers = { + "accept": "application/json", + "content-type": "application/json", + "authorization": "API-Key " + .destination.Config.apiKey + }; + response + + diff --git a/test/integrations/destinations/sprig/processor/data.ts b/test/integrations/destinations/sprig/processor/data.ts new file mode 100644 index 00000000000..f12415c29e9 --- /dev/null +++ b/test/integrations/destinations/sprig/processor/data.ts @@ -0,0 +1,504 @@ +export const data = [ + { + name: 'sprig', + description: 'No message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + userId: 'user@1', + channel: 'web', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + originalTimestamp: '2023-11-10T14:42:44.724Z', + timestamp: '2023-11-22T10:12:44.75705:30', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiKey: 'testApiKey', + }, + }, + metadata: { + jobId: 1, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 1, + }, + statusCode: 400, + error: + 'message Type is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message Type is not present. Aborting', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'SPRIG', + module: 'destination', + implementation: 'cdkV2', + feature: 'processor', + }, + }, + ], + }, + }, + }, + { + name: 'sprig', + description: 'Unsupported message type', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + userId: 'user@1', + groupId: 'group@1', + channel: 'web', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + traits: {}, + type: 'group', + originalTimestamp: '2023-11-10T14:42:44.724Z', + timestamp: '2023-11-22T10:12:44.75705:30', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiKey: 'testApiKey', + }, + }, + metadata: { + jobId: 2, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 2, + }, + statusCode: 400, + error: + 'message type group is not supported: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: message type group is not supported', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'SPRIG', + module: 'destination', + implementation: 'cdkV2', + feature: 'processor', + }, + }, + ], + }, + }, + }, + { + name: 'sprig', + description: 'Missing config', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + userId: 'user@1', + channel: 'web', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + type: 'identify', + originalTimestamp: '2023-11-10T14:42:44.724Z', + timestamp: '2023-11-22T10:12:44.75705:30', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: {}, + }, + metadata: { + jobId: 3, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 3, + }, + statusCode: 400, + error: + 'API Key is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: API Key is not present. Aborting', + statTags: { + errorCategory: 'dataValidation', + errorType: 'configuration', + destType: 'SPRIG', + module: 'destination', + implementation: 'cdkV2', + feature: 'processor', + }, + }, + ], + }, + }, + }, + { + name: 'sprig', + description: 'Identify call without userId', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + anonymousId: 'anon@1', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + type: 'identify', + originalTimestamp: '2023-11-10T14:42:44.724Z', + timestamp: '2023-11-22T10:12:44.75705:30', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiKey: 'testApiKey', + }, + }, + metadata: { + jobId: 4, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 4, + }, + statusCode: 400, + error: + 'userId is required: Workflow: procWorkflow, Step: validateIdentifyPayload, ChildStep: undefined, OriginalError: userId is required', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'SPRIG', + module: 'destination', + implementation: 'cdkV2', + feature: 'processor', + }, + }, + ], + }, + }, + }, + { + name: 'sprig', + description: 'Successful identify call', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + userId: 'user@1', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + type: 'identify', + originalTimestamp: '2023-11-10T14:42:44.724Z', + timestamp: '2023-11-22T10:12:44.75705:30', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiKey: 'testApiKey', + }, + }, + metadata: { + jobId: 5, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 5, + }, + output: { + method: 'POST', + endpoint: 'https://api.sprig.com/v2/users', + headers: { + accept: 'application/json', + authorization: 'API-Key testApiKey', + 'content-type': 'application/json', + }, + body: { + JSON: { + attributes: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + emailAddress: 'rajesh@gmail.com', + userId: 'user@1', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + files: {}, + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'sprig', + description: 'Track call with empty event name', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + userId: 'user@1', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + properties: {}, + type: 'track', + event: '', + originalTimestamp: '2020-11-29T19:11:00.337Z', + timestamp: '2023-11-29T19:11:00.337Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiKey: 'testApiKey', + }, + }, + metadata: { + jobId: 6, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 6, + }, + statusCode: 400, + error: + 'event name is required: Workflow: procWorkflow, Step: validateTrackPayload, ChildStep: undefined, OriginalError: event name is required', + statTags: { + errorCategory: 'dataValidation', + errorType: 'instrumentation', + destType: 'SPRIG', + module: 'destination', + implementation: 'cdkV2', + feature: 'processor', + }, + }, + ], + }, + }, + }, + { + name: 'sprig', + description: 'Successful track call', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + userId: 'user@1', + context: { + traits: { + email: 'rajesh@gmail.com', + firstName: 'Rajesh', + lastName: 'Lodha', + }, + }, + properties: {}, + type: 'track', + event: 'signup', + originalTimestamp: '2020-11-29T19:11:00.337Z', + timestamp: '2023-11-29T19:11:00.337Z', + }, + destination: { + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + apiKey: 'testApiKey', + }, + }, + metadata: { + jobId: 7, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: { + jobId: 7, + }, + output: { + method: 'POST', + endpoint: 'https://api.sprig.com/v2/users', + headers: { + accept: 'application/json', + authorization: 'API-Key testApiKey', + 'content-type': 'application/json', + }, + body: { + JSON: { + emailAddress: 'rajesh@gmail.com', + userId: 'user@1', + events: [ + { + event: 'signup', + timestamp: 1701285060337, + }, + ], + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + files: {}, + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; From a29b37ca11fc625e10fe7198d1b094b7fc7b8768 Mon Sep 17 00:00:00 2001 From: mihir-4116 Date: Thu, 7 Dec 2023 13:38:21 +0530 Subject: [PATCH 2/3] chore: code review changes --- .../destinations/sprig/processor/data.ts | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/test/integrations/destinations/sprig/processor/data.ts b/test/integrations/destinations/sprig/processor/data.ts index f12415c29e9..6b99e5e13b6 100644 --- a/test/integrations/destinations/sprig/processor/data.ts +++ b/test/integrations/destinations/sprig/processor/data.ts @@ -14,9 +14,9 @@ export const data = [ channel: 'web', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Ruddelabs', }, }, originalTimestamp: '2023-11-10T14:42:44.724Z', @@ -80,9 +80,9 @@ export const data = [ channel: 'web', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderstack', }, }, traits: {}, @@ -147,9 +147,9 @@ export const data = [ channel: 'web', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderstack', }, }, type: 'identify', @@ -211,9 +211,9 @@ export const data = [ anonymousId: 'anon@1', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderlabs', }, }, type: 'identify', @@ -277,9 +277,9 @@ export const data = [ userId: 'user@1', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderlabs', }, }, type: 'identify', @@ -323,11 +323,11 @@ export const data = [ body: { JSON: { attributes: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderlabs', }, - emailAddress: 'rajesh@gmail.com', + emailAddress: 'test@gmail.com', userId: 'user@1', }, XML: {}, @@ -361,9 +361,9 @@ export const data = [ userId: 'user@1', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderlabs', }, }, properties: {}, @@ -429,9 +429,9 @@ export const data = [ userId: 'user@1', context: { traits: { - email: 'rajesh@gmail.com', - firstName: 'Rajesh', - lastName: 'Lodha', + email: 'test@gmail.com', + firstName: 'Test', + lastName: 'Rudderlabs', }, }, properties: {}, @@ -476,7 +476,7 @@ export const data = [ }, body: { JSON: { - emailAddress: 'rajesh@gmail.com', + emailAddress: 'test@gmail.com', userId: 'user@1', events: [ { From dd23d9d98b46b5c5fd0e66d6195d8294642e19ac Mon Sep 17 00:00:00 2001 From: mihir-4116 Date: Sat, 9 Dec 2023 09:44:19 +0530 Subject: [PATCH 3/3] chore: code review changes --- .../v2/destinations/sprig/procWorkflow.yaml | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/cdk/v2/destinations/sprig/procWorkflow.yaml b/src/cdk/v2/destinations/sprig/procWorkflow.yaml index 3ad0d7f841f..18b46913fd6 100644 --- a/src/cdk/v2/destinations/sprig/procWorkflow.yaml +++ b/src/cdk/v2/destinations/sprig/procWorkflow.yaml @@ -19,6 +19,12 @@ steps: $.assert(messageType in {{$.EventType.([.IDENTIFY, .TRACK])}}, "message type " + messageType + " is not supported"); $.assertConfig(.destination.Config.apiKey, "API Key is not present. Aborting"); + - name: validateIdentifyPayload + condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} + template: | + const userId = .message.({{{{$.getGenericPaths("userIdOnly")}}}});; + $.assert(userId, "userId is required"); + - name: prepareIdentifyPayload condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} template: | @@ -28,11 +34,13 @@ steps: attributes: .context.traits }); $.context.payload = $.removeUndefinedAndNullValues($.context.payload); - - - name: validateIdentifyPayload - condition: $.outputs.messageType === {{$.EventType.IDENTIFY}} + + - name: validateTrackPayload + condition: $.outputs.messageType === {{$.EventType.TRACK}} template: | - $.assert($.context.payload.userId, "userId is required"); + const userId = .message.({{{{$.getGenericPaths("userIdOnly")}}}});; + $.assert(userId, "userId is required"); + $.assert(.message.event, "event name is required"); - name: prepareTrackPayload condition: $.outputs.messageType === {{$.EventType.TRACK}} @@ -50,12 +58,6 @@ steps: $.context.payload.events = events; $.context.payload = $.removeUndefinedAndNullValues($.context.payload); - - name: validateTrackPayload - condition: $.outputs.messageType === {{$.EventType.TRACK}} - template: | - $.assert($.context.payload.userId, "userId is required"); - $.assert($.context.payload.events[0].event, "event name is required"); - - name: buildResponse template: | const response = $.defaultRequestConfig();