feat: Save the number of successful, failed and total executions for Test Run (no-changelog)

packages/cli/src/databases/entities/test-run.ee.ts

@@ -35,4 +35,23 @@ export class TestRun extends WithTimestampsAndStringId {
 
 	@Column(jsonColumnType, { nullable: true })
 	metrics: AggregatedTestRunMetrics;
+
+	/**
+	 * Total number of the test cases, matching the filter condition of the test definition (specified annotationTag)
+	 */
+	@Column('integer', { nullable: true })
+	totalCases: number;
+
+	/**
+	 * Number of test cases that passed (evaluation workflow was executed successfully)
+	 */
+	@Column('integer', { nullable: true })
+	passedCases: number;
+
+	/**
+	 * Number of failed test cases
+	 * (any unexpected exception happened during the execution or evaluation workflow ended with an error)
+	 */
+	@Column('integer', { nullable: true })
+	failedCases: number;
 }

packages/cli/src/databases/migrations/common/1733494526213-AddStatsColumnsToTestRun.ts

@@ -0,0 +1,29 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+const columns = ['totalCases', 'passedCases', 'failedCases'] as const;
+
+export class AddStatsColumnsToTestRun1733494526213 implements ReversibleMigration {
+	async up({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_run');
+		const columnNames = columns.map((name) => escape.columnName(name));
+
+		// Values can be NULL only if the test run is new, otherwise they must be non-negative integers
+		for (const name of columnNames) {
+			await runQuery(`ALTER TABLE ${tableName} ADD COLUMN ${name} INT CHECK(
+				CASE
+					WHEN status = 'new' THEN ${name} IS NULL
+					ELSE ${name} >= 0
+				END
+			)`);
+		}
+	}
+
+	async down({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_run');
+		const columnNames = columns.map((name) => escape.columnName(name));
+
+		for (const name of columnNames) {
+			await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${name}`);
+		}
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -74,6 +74,7 @@ import { AddDescriptionToTestDefinition1731404028106 } from '../common/173140402
 import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
+import { AddStatsColumnsToTestRun1733494526213 } from '../common/1733494526213-AddStatsColumnsToTestRun';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -150,4 +151,5 @@ export const mysqlMigrations: Migration[] = [
 	CreateTestMetricTable1732271325258,
 	CreateTestRun1732549866705,
 	AddMockedNodesColumnToTestDefinition1733133775640,
+	AddStatsColumnsToTestRun1733494526213,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -74,6 +74,7 @@ import { AddDescriptionToTestDefinition1731404028106 } from '../common/173140402
 import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
+import { AddStatsColumnsToTestRun1733494526213 } from '../common/1733494526213-AddStatsColumnsToTestRun';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -150,4 +151,5 @@ export const postgresMigrations: Migration[] = [
 	CreateTestMetricTable1732271325258,
 	CreateTestRun1732549866705,
 	AddMockedNodesColumnToTestDefinition1733133775640,
+	AddStatsColumnsToTestRun1733494526213,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -71,6 +71,7 @@ import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-
 import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
+import { AddStatsColumnsToTestRun1733494526213 } from '../common/1733494526213-AddStatsColumnsToTestRun';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -144,6 +145,7 @@ const sqliteMigrations: Migration[] = [
 	CreateTestMetricTable1732271325258,
 	CreateTestRun1732549866705,
 	AddMockedNodesColumnToTestDefinition1733133775640,
+	AddStatsColumnsToTestRun1733494526213,
 ];
 
 export { sqliteMigrations };

packages/cli/src/databases/repositories/test-run.repository.ee.ts

@@ -21,14 +21,28 @@ export class TestRunRepository extends Repository<TestRun> {
 		return await this.save(testRun);
 	}
 
-	public async markAsRunning(id: string) {
-		return await this.update(id, { status: 'running', runAt: new Date() });
+	public async markAsRunning(id: string, totalCases: number) {
+		return await this.update(id, {
+			status: 'running',
+			runAt: new Date(),
+			totalCases,
+			passedCases: 0,
+			failedCases: 0,
+		});
 	}
 
 	public async markAsCompleted(id: string, metrics: AggregatedTestRunMetrics) {
 		return await this.update(id, { status: 'completed', completedAt: new Date(), metrics });
 	}
 
+	public async incrementPassed(id: string) {
+		return await this.increment({ id }, 'passedCases', 1);
+	}
+
+	public async incrementFailed(id: string) {
+		return await this.increment({ id }, 'failedCases', 1);
+	}
+
 	public async getMany(testDefinitionId: string, options: ListQuery.Options) {
 		const findManyOptions: FindManyOptions<TestRun> = {
 			where: { testDefinition: { id: testDefinitionId } },

packages/cli/src/evaluation/test-runner/__tests__/test-runner.service.ee.test.ts

@@ -2,7 +2,7 @@ import type { SelectQueryBuilder } from '@n8n/typeorm';
 import { stringify } from 'flatted';
 import { readFileSync } from 'fs';
 import { mock, mockDeep } from 'jest-mock-extended';
-import type { GenericValue, IRun } from 'n8n-workflow';
+import type { ExecutionError, GenericValue, IRun } from 'n8n-workflow';
 import path from 'path';
 
 import type { ActiveExecutions } from '@/active-executions';
@@ -82,6 +82,16 @@ function mockExecutionData() {
 	});
 }
 
+function mockErrorExecutionData() {
+	return mock<IRun>({
+		data: {
+			resultData: {
+				error: mock<ExecutionError>(),
+			},
+		},
+	});
+}
+
 function mockEvaluationExecutionData(metrics: Record<string, GenericValue>) {
 	return mock<IRun>({
 		data: {
@@ -102,6 +112,9 @@ function mockEvaluationExecutionData(metrics: Record<string, GenericValue>) {
 						},
 					],
 				},
+				// error is an optional prop, but jest-mock-extended will mock it by default,
+				// which affects the code logic. So, we need to explicitly set it to undefined.
+				error: undefined,
 			},
 		},
 	});
@@ -148,6 +161,8 @@ describe('TestRunnerService', () => {
 		testRunRepository.createTestRun.mockClear();
 		testRunRepository.markAsRunning.mockClear();
 		testRunRepository.markAsCompleted.mockClear();
+		testRunRepository.incrementFailed.mockClear();
+		testRunRepository.incrementPassed.mockClear();
 	});
 
 	test('should create an instance of TestRunnerService', async () => {
@@ -290,12 +305,182 @@ describe('TestRunnerService', () => {
 		// Check Test Run status was updated correctly
 		expect(testRunRepository.createTestRun).toHaveBeenCalledTimes(1);
 		expect(testRunRepository.markAsRunning).toHaveBeenCalledTimes(1);
-		expect(testRunRepository.markAsRunning).toHaveBeenCalledWith('test-run-id');
+		expect(testRunRepository.markAsRunning).toHaveBeenCalledWith('test-run-id', expect.any(Number));
 		expect(testRunRepository.markAsCompleted).toHaveBeenCalledTimes(1);
 		expect(testRunRepository.markAsCompleted).toHaveBeenCalledWith('test-run-id', {
 			metric1: 0.75,
 			metric2: 0,
 		});
+
+		expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(2);
+		expect(testRunRepository.incrementFailed).not.toHaveBeenCalled();
+	});
+
+	test('should properly count passed and failed executions', async () => {
+		const testRunnerService = new TestRunnerService(
+			workflowRepository,
+			workflowRunner,
+			executionRepository,
+			activeExecutions,
+			testRunRepository,
+			testMetricRepository,
+			mockNodeTypes,
+		);
+
+		workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({
+			id: 'workflow-under-test-id',
+			...wfUnderTestJson,
+		});
+
+		workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({
+			id: 'evaluation-workflow-id',
+			...wfEvaluationJson,
+		});
+
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-2');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-3');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-4');
+
+		// Mock executions of workflow under test
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id')
+			.mockResolvedValue(mockExecutionData());
+
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-3')
+			.mockResolvedValue(mockExecutionData());
+
+		// Mock executions of evaluation workflow
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-2')
+			.mockResolvedValue(mockEvaluationExecutionData({ metric1: 1, metric2: 0 }));
+
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-4')
+			.mockRejectedValue(new Error('Some error'));
+
+		await testRunnerService.runTest(
+			mock<User>(),
+			mock<TestDefinition>({
+				workflowId: 'workflow-under-test-id',
+				evaluationWorkflowId: 'evaluation-workflow-id',
+				mockedNodes: [],
+			}),
+		);
+
+		expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(1);
+		expect(testRunRepository.incrementFailed).toHaveBeenCalledTimes(1);
+	});
+
+	test('should properly count failed test executions', async () => {
+		const testRunnerService = new TestRunnerService(
+			workflowRepository,
+			workflowRunner,
+			executionRepository,
+			activeExecutions,
+			testRunRepository,
+			testMetricRepository,
+			mockNodeTypes,
+		);
+
+		workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({
+			id: 'workflow-under-test-id',
+			...wfUnderTestJson,
+		});
+
+		workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({
+			id: 'evaluation-workflow-id',
+			...wfEvaluationJson,
+		});
+
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-2');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-3');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-4');
+
+		// Mock executions of workflow under test
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id')
+			.mockResolvedValue(mockExecutionData());
+
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-3')
+			.mockResolvedValue(mockErrorExecutionData());
+
+		// Mock executions of evaluation workflow
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-2')
+			.mockResolvedValue(mockEvaluationExecutionData({ metric1: 1, metric2: 0 }));
+
+		await testRunnerService.runTest(
+			mock<User>(),
+			mock<TestDefinition>({
+				workflowId: 'workflow-under-test-id',
+				evaluationWorkflowId: 'evaluation-workflow-id',
+				mockedNodes: [],
+			}),
+		);
+
+		expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(1);
+		expect(testRunRepository.incrementFailed).toHaveBeenCalledTimes(1);
+	});
+
+	test('should properly count failed evaluations', async () => {
+		const testRunnerService = new TestRunnerService(
+			workflowRepository,
+			workflowRunner,
+			executionRepository,
+			activeExecutions,
+			testRunRepository,
+			testMetricRepository,
+			mockNodeTypes,
+		);
+
+		workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({
+			id: 'workflow-under-test-id',
+			...wfUnderTestJson,
+		});
+
+		workflowRepository.findById.calledWith('evaluation-workflow-id').mockResolvedValueOnce({
+			id: 'evaluation-workflow-id',
+			...wfEvaluationJson,
+		});
+
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-2');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-3');
+		workflowRunner.run.mockResolvedValueOnce('some-execution-id-4');
+
+		// Mock executions of workflow under test
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id')
+			.mockResolvedValue(mockExecutionData());
+
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-3')
+			.mockResolvedValue(mockExecutionData());
+
+		// Mock executions of evaluation workflow
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-2')
+			.mockResolvedValue(mockEvaluationExecutionData({ metric1: 1, metric2: 0 }));
+
+		activeExecutions.getPostExecutePromise
+			.calledWith('some-execution-id-4')
+			.mockResolvedValue(mockErrorExecutionData());
+
+		await testRunnerService.runTest(
+			mock<User>(),
+			mock<TestDefinition>({
+				workflowId: 'workflow-under-test-id',
+				evaluationWorkflowId: 'evaluation-workflow-id',
+				mockedNodes: [],
+			}),
+		);
+
+		expect(testRunRepository.incrementPassed).toHaveBeenCalledTimes(1);
+		expect(testRunRepository.incrementFailed).toHaveBeenCalledTimes(1);
 	});
 
 	test('should specify correct start nodes when running workflow under test', async () => {