feat(core): Keep track of test case executions during test run (no-changelog)

packages/cli/src/databases/entities/index.ts

@@ -20,6 +20,7 @@ import { Settings } from './settings';
 import { SharedCredentials } from './shared-credentials';
 import { SharedWorkflow } from './shared-workflow';
 import { TagEntity } from './tag-entity';
+import { TestCaseExecution } from './test-case-execution.ee';
 import { TestDefinition } from './test-definition.ee';
 import { TestMetric } from './test-metric.ee';
 import { TestRun } from './test-run.ee';
@@ -64,4 +65,5 @@ export const entities = {
 	TestDefinition,
 	TestMetric,
 	TestRun,
+	TestCaseExecution,
 };

packages/cli/src/databases/entities/test-case-execution.ee.ts

@@ -0,0 +1,68 @@
+import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm';
+
+import {
+	datetimeColumnType,
+	jsonColumnType,
+	WithStringId,
+} from '@/databases/entities/abstract-entity';
+import type { ExecutionEntity } from '@/databases/entities/execution-entity';
+import { TestRun } from '@/databases/entities/test-run.ee';
+
+export type TestCaseRunMetrics = Record<string, number | boolean>;
+
+/**
+ * This entity represents the linking between the test runs and individual executions.
+ * It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions
+ * Entries in this table are meant to outlive the execution entities, which might be pruned over time.
+ * This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted.
+ */
+@Entity({ name: 'test_case_execution' })
+export class TestCaseExecution extends WithStringId {
+	@ManyToOne('TestRun')
+	testRun: TestRun;
+
+	@ManyToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	pastExecution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	pastExecutionId: string | null;
+
+	@OneToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	execution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	executionId: string | null;
+
+	@OneToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	evaluationExecution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	evaluationExecutionId: string | null;
+
+	@Column()
+	status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled';
+
+	@Column({ type: datetimeColumnType, nullable: true })
+	runAt: Date | null;
+
+	@Column({ type: datetimeColumnType, nullable: true })
+	completedAt: Date | null;
+
+	@Column('varchar', { nullable: true })
+	errorCode: string | null;
+
+	@Column(jsonColumnType, { nullable: true })
+	errorDetails: Record<string, unknown>;
+
+	@Column(jsonColumnType, { nullable: true })
+	metrics: TestCaseRunMetrics;
+}

packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts

@@ -0,0 +1,47 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+const testCaseExecutionTableName = 'test_case_execution';
+
+export class CreateTestCaseExecutionTable1736947513045 implements ReversibleMigration {
+	async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
+		await createTable(testCaseExecutionTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('testRunId').varchar(36).notNull,
+				column('pastExecutionId').int, // Might be null if execution was deleted after the test run
+				column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run
+				column('evaluationExecutionId').int, // Execution of the evaluation workflow. Might be null if execution was deleted after the test run, or if the test run was cancelled
+				column('status').varchar().notNull,
+				column('runAt').timestamp(),
+				column('completedAt').timestamp(),
+				column('errorCode').varchar(),
+				column('errorDetails').json,
+				column('metrics').json,
+			)
+			.withIndexOn('testRunId')
+			.withForeignKey('testRunId', {
+				tableName: 'test_run',
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withForeignKey('pastExecutionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			})
+			.withForeignKey('executionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			})
+			.withForeignKey('evaluationExecutionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			}).withTimestamps;
+	}
+
+	async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+		await dropTable(testCaseExecutionTableName);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -156,4 +157,5 @@ export const mysqlMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -156,4 +157,5 @@ export const postgresMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -74,6 +74,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -150,6 +151,7 @@ const sqliteMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];
 
 export { sqliteMigrations };

packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts

@@ -0,0 +1,99 @@
+import { Service } from '@n8n/di';
+import type { EntityManager } from '@n8n/typeorm';
+import { DataSource, In, Not, Repository } from '@n8n/typeorm';
+import type { DeepPartial } from '@n8n/typeorm/common/DeepPartial';
+
+import { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
+
+@Service()
+export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
+	constructor(dataSource: DataSource) {
+		super(TestCaseExecution, dataSource.manager);
+	}
+
+	async createBatch(testRunId: string, pastExecutionIds: string[]) {
+		const mappings = this.create(
+			pastExecutionIds.map<DeepPartial<TestCaseExecution>>((id) => ({
+				testRun: {
+					id: testRunId,
+				},
+				pastExecution: {
+					id,
+				},
+				status: 'new',
+			})),
+		);
+
+		return await this.save(mappings);
+	}
+
+	async markAsRunning(testRunId: string, pastExecutionId: string, executionId: string) {
+		return await this.update(
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'running',
+				executionId,
+				runAt: new Date(),
+			},
+		);
+	}
+
+	async markAsEvaluationRunning(
+		testRunId: string,
+		pastExecutionId: string,
+		evaluationExecutionId: string,
+	) {
+		return await this.update(
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'evaluation_running',
+				evaluationExecutionId,
+			},
+		);
+	}
+
+	async markAsCompleted(
+		testRunId: string,
+		pastExecutionId: string,
+		metrics: Record<string, number>,
+		trx?: EntityManager,
+	) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'success',
+				completedAt: new Date(),
+				metrics,
+			},
+		);
+	}
+
+	async markAllPendingAsCancelled(testRunId: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, status: Not(In(['success', 'error', 'cancelled'])) },
+			{
+				status: 'cancelled',
+				completedAt: new Date(),
+			},
+		);
+	}
+
+	async markAsFailed(testRunId: string, pastExecutionId: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'error',
+				completedAt: new Date(),
+			},
+		);
+	}
+}

packages/cli/src/databases/repositories/test-run.repository.ee.ts

@@ -1,5 +1,5 @@
 import { Service } from '@n8n/di';
-import type { FindManyOptions } from '@n8n/typeorm';
+import type { EntityManager, FindManyOptions } from '@n8n/typeorm';
 import { DataSource, Repository } from '@n8n/typeorm';
 
 import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee';
@@ -35,16 +35,19 @@ export class TestRunRepository extends Repository<TestRun> {
 		return await this.update(id, { status: 'completed', completedAt: new Date(), metrics });
 	}
 
-	async markAsCancelled(id: string) {
-		return await this.update(id, { status: 'cancelled' });
+	async markAsCancelled(id: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+		return await trx.update(TestRun, id, { status: 'cancelled' });
 	}
 
-	async incrementPassed(id: string) {
-		return await this.increment({ id }, 'passedCases', 1);
+	async incrementPassed(id: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+		return await trx.increment(TestRun, { id }, 'passedCases', 1);
 	}
 
-	async incrementFailed(id: string) {
-		return await this.increment({ id }, 'failedCases', 1);
+	async incrementFailed(id: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+		return await trx.increment(TestRun, { id }, 'failedCases', 1);
 	}
 
 	async getMany(testDefinitionId: string, options: ListQuery.Options) {

packages/cli/src/evaluation.ee/test-definitions.types.ee.ts

@@ -94,4 +94,6 @@ export declare namespace TestRunsRequest {
 	type Delete = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
 
 	type Cancel = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
+
+	type GetCases = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
 }