diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 4bd1890c4e21b..abcf298d3d5e8 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -195,4 +195,3 @@ export const WsStatusCodes = { } as const; export const FREE_AI_CREDITS_CREDENTIAL_NAME = 'n8n free OpenAI API credits'; -export const OPEN_AI_API_CREDENTIAL_TYPE = 'openAiApi'; diff --git a/packages/cli/src/controllers/ai.controller.ts b/packages/cli/src/controllers/ai.controller.ts index 791c02bec3470..e82ecd77efcb3 100644 --- a/packages/cli/src/controllers/ai.controller.ts +++ b/packages/cli/src/controllers/ai.controller.ts @@ -6,10 +6,11 @@ import { } from '@n8n/api-types'; import type { AiAssistantSDK } from '@n8n_io/ai-assistant-sdk'; import { Response } from 'express'; +import { OPEN_AI_API_CREDENTIAL_TYPE } from 'n8n-workflow'; import { strict as assert } from 'node:assert'; import { WritableStream } from 'node:stream/web'; -import { FREE_AI_CREDITS_CREDENTIAL_NAME, OPEN_AI_API_CREDENTIAL_TYPE } from '@/constants'; +import { FREE_AI_CREDITS_CREDENTIAL_NAME } from '@/constants'; import { CredentialsService } from '@/credentials/credentials.service'; import { Body, Post, RestController } from '@/decorators'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts index c41ea773dd273..01673cd375bb6 100644 --- a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -1622,5 +1622,74 @@ describe('TelemetryEventRelay', () => { }), ); }); + + it('should call telemetry.track when user ran out of free AI credits', async () => { + sharedWorkflowRepository.findSharingRole.mockResolvedValue('workflow:editor'); + credentialsRepository.findOneBy.mockResolvedValue( + mock({ type: 'openAiApi', isManaged: true }), + ); + + const runData = { + status: 'error', + mode: 'trigger', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'OpenAI', + type: 'n8n-nodes-base.openAi', + parameters: {}, + position: [100, 200], + }, + { + message: `400 - ${JSON.stringify({ + error: { + message: 'error message', + type: 'error_type', + code: 200, + }, + })}`, + error: { + message: 'error message', + type: 'error_type', + code: 200, + }, + }, + { + httpCode: '400', + }, + ), + }, + }, + } as unknown as IRun; + + jest + .spyOn(TelemetryHelpers, 'userInInstanceRanOutOfFreeAiCredits') + .mockImplementation(() => true); + + const event: RelayEventMap['workflow-post-execute'] = { + workflow: mockWorkflowBase, + executionId: 'execution123', + userId: 'user123', + runData, + }; + + eventService.emit('workflow-post-execute', event); + + await flushPromises(); + + expect(telemetry.track).toHaveBeenCalledWith('User ran out of free AI credits'); + }); }); }); diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index d2bc61c733943..67fbacb107d13 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -634,6 +634,10 @@ export class TelemetryEventRelay extends EventRelay { let nodeGraphResult: INodesGraphResult | null = null; if (!telemetryProperties.success && runData?.data.resultData.error) { + if (TelemetryHelpers.userInInstanceRanOutOfFreeAiCredits(runData)) { + this.telemetry.track('User ran out of free AI credits'); + } + telemetryProperties.error_message = runData?.data.resultData.error.message; let errorNodeName = 'node' in runData?.data.resultData.error diff --git a/packages/cli/test/integration/ai/ai.api.test.ts b/packages/cli/test/integration/ai/ai.api.test.ts index 721f2296ed5ce..6741930b2e53c 100644 --- a/packages/cli/test/integration/ai/ai.api.test.ts +++ b/packages/cli/test/integration/ai/ai.api.test.ts @@ -1,8 +1,9 @@ import { Container } from '@n8n/di'; import { randomUUID } from 'crypto'; import { mock } from 'jest-mock-extended'; +import { OPEN_AI_API_CREDENTIAL_TYPE } from 'n8n-workflow'; -import { FREE_AI_CREDITS_CREDENTIAL_NAME, OPEN_AI_API_CREDENTIAL_TYPE } from '@/constants'; +import { FREE_AI_CREDITS_CREDENTIAL_NAME } from '@/constants'; import type { Project } from '@/databases/entities/project'; import type { User } from '@/databases/entities/user'; import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; diff --git a/packages/editor-ui/src/components/FreeAiCreditsCallout.vue b/packages/editor-ui/src/components/FreeAiCreditsCallout.vue index 922df4cf89c85..97dd4eb9dbea8 100644 --- a/packages/editor-ui/src/components/FreeAiCreditsCallout.vue +++ b/packages/editor-ui/src/components/FreeAiCreditsCallout.vue @@ -10,8 +10,7 @@ import { useProjectsStore } from '@/stores/projects.store'; import { useSettingsStore } from '@/stores/settings.store'; import { useUsersStore } from '@/stores/users.store'; import { computed, ref } from 'vue'; - -const OPEN_AI_API_CREDENTIAL_TYPE = 'openAiApi'; +import { OPEN_AI_API_CREDENTIAL_TYPE } from 'n8n-workflow'; const LANGCHAIN_NODES_PREFIX = '@n8n/n8n-nodes-langchain.'; diff --git a/packages/workflow/src/Constants.ts b/packages/workflow/src/Constants.ts index e5983013deb40..453b9be7484c0 100644 --- a/packages/workflow/src/Constants.ts +++ b/packages/workflow/src/Constants.ts @@ -95,3 +95,7 @@ export const AI_TRANSFORM_JS_CODE = 'jsCode'; * in `cli` package. */ export const TRIMMED_TASK_DATA_CONNECTIONS_KEY = '__isTrimmedManualExecutionDataItem'; + +export const OPEN_AI_API_CREDENTIAL_TYPE = 'openAiApi'; +export const FREE_AI_CREDITS_ERROR_TYPE = 'free_ai_credits_request_error'; +export const FREE_AI_CREDITS_USED_ALL_CREDITS_ERROR_CODE = 400; diff --git a/packages/workflow/src/TelemetryHelpers.ts b/packages/workflow/src/TelemetryHelpers.ts index 03d675e0e515d..51d56ef15459c 100644 --- a/packages/workflow/src/TelemetryHelpers.ts +++ b/packages/workflow/src/TelemetryHelpers.ts @@ -4,16 +4,20 @@ import { CHAIN_LLM_LANGCHAIN_NODE_TYPE, CHAIN_SUMMARIZATION_LANGCHAIN_NODE_TYPE, EXECUTE_WORKFLOW_NODE_TYPE, + FREE_AI_CREDITS_ERROR_TYPE, + FREE_AI_CREDITS_USED_ALL_CREDITS_ERROR_CODE, HTTP_REQUEST_NODE_TYPE, HTTP_REQUEST_TOOL_LANGCHAIN_NODE_TYPE, LANGCHAIN_CUSTOM_TOOLS, MERGE_NODE_TYPE, + OPEN_AI_API_CREDENTIAL_TYPE, OPENAI_LANGCHAIN_NODE_TYPE, STICKY_NODE_TYPE, WEBHOOK_NODE_TYPE, WORKFLOW_TOOL_LANGCHAIN_NODE_TYPE, } from './Constants'; import { ApplicationError } from './errors/application.error'; +import type { NodeApiError } from './errors/node-api.error'; import type { IConnection, INode, @@ -29,6 +33,10 @@ import type { IRun, } from './Interfaces'; import { getNodeParameters } from './NodeHelpers'; +import { jsonParse } from './utils'; + +const isNodeApiError = (error: unknown): error is NodeApiError => + typeof error === 'object' && error !== null && 'name' in error && error?.name === 'NodeApiError'; export function getNodeTypeForName(workflow: IWorkflowBase, nodeName: string): INode | undefined { return workflow.nodes.find((node) => node.name === nodeName); @@ -489,3 +497,31 @@ export function extractLastExecutedNodeCredentialData( return { credentialId: id, credentialType }; } + +export const userInInstanceRanOutOfFreeAiCredits = (runData: IRun): boolean => { + const credentials = extractLastExecutedNodeCredentialData(runData); + + if (!credentials) return false; + + if (credentials.credentialType !== OPEN_AI_API_CREDENTIAL_TYPE) return false; + + const { error } = runData.data.resultData; + + if (!isNodeApiError(error) || !error.messages[0]) return false; + + const rawErrorResponse = error.messages[0].replace(`${error.httpCode} -`, ''); + + try { + const errorResponse = jsonParse<{ error: { code: number; type: string } }>(rawErrorResponse); + if ( + errorResponse?.error?.type === FREE_AI_CREDITS_ERROR_TYPE && + errorResponse.error.code === FREE_AI_CREDITS_USED_ALL_CREDITS_ERROR_CODE + ) { + return true; + } + } catch { + return false; + } + + return false; +}; diff --git a/packages/workflow/test/TelemetryHelpers.test.ts b/packages/workflow/test/TelemetryHelpers.test.ts index c2b15864f5ddc..4cb923bb9d627 100644 --- a/packages/workflow/test/TelemetryHelpers.test.ts +++ b/packages/workflow/test/TelemetryHelpers.test.ts @@ -2,7 +2,7 @@ import { mock } from 'jest-mock-extended'; import { v5 as uuidv5, v3 as uuidv3, v4 as uuidv4, v1 as uuidv1 } from 'uuid'; import { STICKY_NODE_TYPE } from '@/Constants'; -import { ApplicationError } from '@/errors'; +import { ApplicationError, ExpressionError, NodeApiError } from '@/errors'; import type { IRun, IRunData } from '@/Interfaces'; import { NodeConnectionType, type IWorkflowBase } from '@/Interfaces'; import * as nodeHelpers from '@/NodeHelpers'; @@ -12,6 +12,7 @@ import { generateNodesGraph, getDomainBase, getDomainPath, + userInInstanceRanOutOfFreeAiCredits, } from '@/TelemetryHelpers'; import { randomInt } from '@/utils'; @@ -930,6 +931,227 @@ describe('extractLastExecutedNodeCredentialData', () => { }); }); +describe('userInInstanceRanOutOfFreeAiCredits', () => { + it('should return false if could not find node credentials', () => { + const runData = { + status: 'error', + mode: 'manual', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: {} } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'OpenAI', + type: 'n8n-nodes-base.openAi', + parameters: {}, + position: [100, 200], + }, + { + message: `400 - ${JSON.stringify({ + error: { + message: 'error message', + type: 'free_ai_credits_request_error', + code: 200, + }, + })}`, + error: { + message: 'error message', + type: 'free_ai_credits_request_error', + code: 200, + }, + }, + { + httpCode: '400', + }, + ), + }, + }, + } as unknown as IRun; + + expect(userInInstanceRanOutOfFreeAiCredits(runData)).toBe(false); + }); + + it('should return false if could not credential type it is not openAiApi', () => { + const runData = { + status: 'error', + mode: 'manual', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: { jiraApi: { id: 'nhu-l8E4hX' } } } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'OpenAI', + type: 'n8n-nodes-base.openAi', + parameters: {}, + position: [100, 200], + }, + { + message: `400 - ${JSON.stringify({ + error: { + message: 'error message', + type: 'free_ai_credits_request_error', + code: 200, + }, + })}`, + error: { + message: 'error message', + type: 'free_ai_credits_request_error', + code: 200, + }, + }, + { + httpCode: '400', + }, + ), + }, + }, + } as unknown as IRun; + + expect(userInInstanceRanOutOfFreeAiCredits(runData)).toBe(false); + }); + + it('should return false if error is not NodeApiError', () => { + const runData = { + status: 'error', + mode: 'manual', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new ExpressionError('error'), + }, + }, + } as unknown as IRun; + + expect(userInInstanceRanOutOfFreeAiCredits(runData)).toBe(false); + }); + + it('should return false if error is not a free ai credit error', () => { + const runData = { + status: 'error', + mode: 'manual', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'OpenAI', + type: 'n8n-nodes-base.openAi', + parameters: {}, + position: [100, 200], + }, + { + message: `400 - ${JSON.stringify({ + error: { + message: 'error message', + type: 'error_type', + code: 200, + }, + })}`, + error: { + message: 'error message', + type: 'error_type', + code: 200, + }, + }, + { + httpCode: '400', + }, + ), + }, + }, + } as unknown as IRun; + + expect(userInInstanceRanOutOfFreeAiCredits(runData)).toBe(false); + }); + + it('should return true if the user has ran out of free AI credits', () => { + const runData = { + status: 'error', + mode: 'manual', + data: { + startData: { + destinationNode: 'OpenAI', + runNodeFilter: ['OpenAI'], + }, + executionData: { + nodeExecutionStack: [{ node: { credentials: { openAiApi: { id: 'nhu-l8E4hX' } } } }], + }, + resultData: { + runData: {}, + lastNodeExecuted: 'OpenAI', + error: new NodeApiError( + { + id: '1', + typeVersion: 1, + name: 'OpenAI', + type: 'n8n-nodes-base.openAi', + parameters: {}, + position: [100, 200], + }, + { + message: `400 - ${JSON.stringify({ + error: { + message: 'error message', + type: 'free_ai_credits_request_error', + code: 400, + }, + })}`, + error: { + message: 'error message', + type: 'free_ai_credits_request_error', + code: 400, + }, + }, + { + httpCode: '400', + }, + ), + }, + }, + } as unknown as IRun; + + expect(userInInstanceRanOutOfFreeAiCredits(runData)).toBe(true); + }); +}); + function validUrls(idMaker: typeof alphanumericId | typeof email, char = CHAR) { const firstId = idMaker(); const secondId = idMaker();