diff --git a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts index cc3c3bc33b137..5d8fe3fe1038a 100644 --- a/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts +++ b/packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts @@ -19,7 +19,7 @@ import type { WorkflowRepository } from '@/databases/repositories/workflow.repos import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { NodeTypes } from '@/node-types'; import type { WorkflowRunner } from '@/workflow-runner'; -import { mockInstance } from '@test/mocking'; +import { mockInstance, mockLogger } from '@test/mocking'; import { mockNodeTypesData } from '@test-integration/utils/node-types-data'; import { TestRunnerService } from '../test-runner.service.ee'; @@ -129,6 +129,9 @@ function mockEvaluationExecutionData(metrics: Record) { }); } +const errorReporter = mock(); +const logger = mockLogger(); + describe('TestRunnerService', () => { const executionRepository = mock(); const workflowRepository = mock(); @@ -176,6 +179,7 @@ describe('TestRunnerService', () => { test('should create an instance of TestRunnerService', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -183,7 +187,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); expect(testRunnerService).toBeInstanceOf(TestRunnerService); @@ -191,6 +195,7 @@ describe('TestRunnerService', () => { test('should create and run test cases from past executions', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -198,7 +203,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -229,6 +234,7 @@ describe('TestRunnerService', () => { test('should run both workflow under test and evaluation workflow', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -236,7 +242,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -330,6 +336,7 @@ describe('TestRunnerService', () => { test('should properly count passed and failed executions', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -337,7 +344,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -388,6 +395,7 @@ describe('TestRunnerService', () => { test('should properly count failed test executions', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -395,7 +403,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -442,6 +450,7 @@ describe('TestRunnerService', () => { test('should properly count failed evaluations', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -449,7 +458,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -500,6 +509,7 @@ describe('TestRunnerService', () => { test('should specify correct start nodes when running workflow under test', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -507,7 +517,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({ @@ -574,6 +584,7 @@ describe('TestRunnerService', () => { test('should properly choose trigger and start nodes', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -581,7 +592,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); const startNodesData = (testRunnerService as any).getStartNodesData( @@ -599,6 +610,7 @@ describe('TestRunnerService', () => { test('should properly choose trigger and start nodes 2', async () => { const testRunnerService = new TestRunnerService( + logger, workflowRepository, workflowRunner, executionRepository, @@ -606,7 +618,7 @@ describe('TestRunnerService', () => { testRunRepository, testMetricRepository, mockNodeTypes, - mock(), + errorReporter, ); const startNodesData = (testRunnerService as any).getStartNodesData( diff --git a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts index 5a054a352765a..732c803814140 100644 --- a/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts +++ b/packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts @@ -1,6 +1,6 @@ import { Service } from '@n8n/di'; import { parse } from 'flatted'; -import { ErrorReporter } from 'n8n-core'; +import { ErrorReporter, Logger } from 'n8n-core'; import { NodeConnectionType, Workflow } from 'n8n-workflow'; import type { IDataObject, @@ -39,6 +39,7 @@ import { createPinData, getPastExecutionTriggerNode } from './utils.ee'; @Service() export class TestRunnerService { constructor( + private readonly logger: Logger, private readonly workflowRepository: WorkflowRepository, private readonly workflowRunner: WorkflowRunner, private readonly executionRepository: ExecutionRepository, @@ -115,8 +116,9 @@ export class TestRunnerService { executionMode: 'evaluation', runData: {}, pinData, - workflowData: workflow, + workflowData: { ...workflow, pinData }, userId, + partialExecutionVersion: '1', }; // Trigger the workflow under test with mocked data @@ -203,6 +205,8 @@ export class TestRunnerService { * Creates a new test run for the given test definition. */ async runTest(user: User, test: TestDefinition): Promise { + this.logger.debug('Starting new test run', { testId: test.id }); + const workflow = await this.workflowRepository.findById(test.workflowId); assert(workflow, 'Workflow not found'); @@ -227,6 +231,8 @@ export class TestRunnerService { .andWhere('execution.workflowId = :workflowId', { workflowId: test.workflowId }) .getMany(); + this.logger.debug('Found past executions', { count: pastExecutions.length }); + // Get the metrics to collect from the evaluation workflow const testMetricNames = await this.getTestMetricNames(test.id); @@ -238,6 +244,8 @@ export class TestRunnerService { const metrics = new EvaluationMetrics(testMetricNames); for (const { id: pastExecutionId } of pastExecutions) { + this.logger.debug('Running test case', { pastExecutionId }); + try { // Fetch past execution with data const pastExecution = await this.executionRepository.findOne({ @@ -257,6 +265,8 @@ export class TestRunnerService { user.id, ); + this.logger.debug('Test case execution finished', { pastExecutionId }); + // In case of a permission check issue, the test case execution will be undefined. // Skip them, increment the failed count and continue with the next test case if (!testCaseExecution) { @@ -279,6 +289,8 @@ export class TestRunnerService { ); assert(evalExecution); + this.logger.debug('Evaluation execution finished', { pastExecutionId }); + metrics.addResults(this.extractEvaluationResult(evalExecution)); if (evalExecution.data.resultData.error) { @@ -297,5 +309,7 @@ export class TestRunnerService { const aggregatedMetrics = metrics.getAggregatedMetrics(); await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics); + + this.logger.debug('Test run finished', { testId: test.id }); } } diff --git a/packages/cli/src/manual-execution.service.ts b/packages/cli/src/manual-execution.service.ts index fd6c27215f082..0b8b9360d39ae 100644 --- a/packages/cli/src/manual-execution.service.ts +++ b/packages/cli/src/manual-execution.service.ts @@ -71,7 +71,11 @@ export class ManualExecutionService { }, }; - const workflowExecute = new WorkflowExecute(additionalData, 'manual', executionData); + const workflowExecute = new WorkflowExecute( + additionalData, + data.executionMode, + executionData, + ); return workflowExecute.processRunExecutionData(workflow); } else if ( data.runData === undefined || diff --git a/packages/cli/src/webhooks/webhook-helpers.ts b/packages/cli/src/webhooks/webhook-helpers.ts index 1711d18056952..dd6a77560686e 100644 --- a/packages/cli/src/webhooks/webhook-helpers.ts +++ b/packages/cli/src/webhooks/webhook-helpers.ts @@ -454,7 +454,7 @@ export async function executeWebhook( } let pinData: IPinData | undefined; - const usePinData = executionMode === 'manual'; + const usePinData = ['manual', 'evaluation'].includes(executionMode); if (usePinData) { pinData = workflowData.pinData; runExecutionData.resultData.pinData = pinData; diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index a5ffb728d62c9..598e6a8b58fed 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -234,7 +234,7 @@ export class WorkflowRunner { } let pinData: IPinData | undefined; - if (data.executionMode === 'manual') { + if (['manual', 'evaluation'].includes(data.executionMode)) { pinData = data.pinData ?? data.workflowData.pinData; } diff --git a/packages/editor-ui/src/composables/useCanvasOperations.ts b/packages/editor-ui/src/composables/useCanvasOperations.ts index dacac9cd2cfdf..46cfc540b8b4b 100644 --- a/packages/editor-ui/src/composables/useCanvasOperations.ts +++ b/packages/editor-ui/src/composables/useCanvasOperations.ts @@ -1923,7 +1923,7 @@ export function useCanvasOperations({ router }: { router: ReturnType