fix TestScheduler dispatch upon exec error

.vscode/settings.json

@@ -5,5 +5,9 @@
   },
   "javascript.validate.enable": false,
   "jest.jestCommandLine": "yarn jest",
-  "typescript.tsdk": "node_modules/typescript/lib"
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "jest.autoRun": {
+    "watch": false,
+    "onSave": "test-src-file",
+  }
 }

CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Fixes
 
 - `[babel-plugin-jest-hoist]` Support imported `jest` in mock factory ([#13188](https://github.com/facebook/jest/pull/13188))
+- `[jest-core]` Capture execError during `TestScheduler.scheduleTests` and dispatch to reporters ([#13203](https://github.com/facebook/jest/pull/13203))
 - `[jest-runtime]` Support `jest.resetModules()` with ESM ([#13211](https://github.com/facebook/jest/pull/13211))
 
 ### Chore & Maintenance

packages/jest-core/src/TestScheduler.ts

@@ -31,13 +31,13 @@ import {
 } from '@jest/test-result';
 import {createScriptTransformer} from '@jest/transform';
 import type {Config} from '@jest/types';
-import {formatExecError} from 'jest-message-util';
+import {formatExecError, separateMessageFromStack} from 'jest-message-util';
 import type {JestTestRunner, TestRunnerContext} from 'jest-runner';
 import {
   buildSnapshotResolver,
   cleanup as cleanupSnapshots,
 } from 'jest-snapshot';
-import {requireOrImportModule} from 'jest-util';
+import {ErrorWithStack, requireOrImportModule} from 'jest-util';
 import type {TestWatcher} from 'jest-watcher';
 import ReporterDispatcher from './ReporterDispatcher';
 import {shouldRunInBand} from './testSchedulerHelper';
@@ -201,6 +201,27 @@ class TestScheduler {
           aggregatedResults.snapshot.filesRemoved)
       );
     };
+    const buildExecError = (err: unknown): SerializableError => {
+      const fromString = (errString: string): SerializableError => {
+        const {message, stack} = separateMessageFromStack(errString);
+        if (stack.length > 0) {
+          return {message, stack};
+        }
+        const error = new ErrorWithStack(message, buildExecError);
+        return {message, stack: error.stack || ''};
+      };
+      if (typeof err === 'string' || err == null) {
+        return fromString(err || 'Error');
+      }
+      const anyErr: any = err as any;
+      if (typeof anyErr.message === 'string') {
+        if (typeof anyErr.stack === 'string' && anyErr.stack.length > 0) {
+          return anyErr;
+        }
+        return fromString(anyErr.message);
+      }
+      return fromString(JSON.stringify(err));
+    };
 
     await this._dispatcher.onRunStart(aggregatedResults, {
       estimatedTime,
@@ -209,79 +230,86 @@ class TestScheduler {
 
     const testRunners: Record<string, JestTestRunner> = Object.create(null);
     const contextsByTestRunner = new WeakMap<JestTestRunner, TestContext>();
-    await Promise.all(
-      Array.from(testContexts).map(async context => {
-        const {config} = context;
-        if (!testRunners[config.runner]) {
-          const transformer = await createScriptTransformer(config);
-          const Runner: TestRunnerConstructor =
-            await transformer.requireAndTranspileModule(config.runner);
-          const runner = new Runner(this._globalConfig, {
-            changedFiles: this._context.changedFiles,
-            sourcesRelatedToTestsInChangedFiles:
-              this._context.sourcesRelatedToTestsInChangedFiles,
-          });
-          testRunners[config.runner] = runner;
-          contextsByTestRunner.set(runner, context);
-        }
-      }),
-    );
 
-    const testsByRunner = this._partitionTests(testRunners, tests);
+    try {
+      await Promise.all(
+        Array.from(testContexts).map(async context => {
+          const {config} = context;
+          if (!testRunners[config.runner]) {
+            const transformer = await createScriptTransformer(config);
+            const Runner: TestRunnerConstructor =
+              await transformer.requireAndTranspileModule(config.runner);
+            const runner = new Runner(this._globalConfig, {
+              changedFiles: this._context.changedFiles,
+              sourcesRelatedToTestsInChangedFiles:
+                this._context.sourcesRelatedToTestsInChangedFiles,
+            });
+            testRunners[config.runner] = runner;
+            contextsByTestRunner.set(runner, context);
+          }
+        }),
+      );
 
-    if (testsByRunner) {
-      try {
-        for (const runner of Object.keys(testRunners)) {
-          const testRunner = testRunners[runner];
-          const context = contextsByTestRunner.get(testRunner);
-
-          invariant(context);
-
-          const tests = testsByRunner[runner];
-
-          const testRunnerOptions = {
-            serial: runInBand || Boolean(testRunner.isSerial),
-          };
-
-          if (testRunner.supportsEventEmitters) {
-            const unsubscribes = [
-              testRunner.on('test-file-start', ([test]) =>
-                onTestFileStart(test),
-              ),
-              testRunner.on('test-file-success', ([test, testResult]) =>
-                onResult(test, testResult),
-              ),
-              testRunner.on('test-file-failure', ([test, error]) =>
-                onFailure(test, error),
-              ),
-              testRunner.on(
-                'test-case-result',
-                ([testPath, testCaseResult]) => {
-                  const test: Test = {context, path: testPath};
-                  this._dispatcher.onTestCaseResult(test, testCaseResult);
-                },
-              ),
-            ];
-
-            await testRunner.runTests(tests, watcher, testRunnerOptions);
-
-            unsubscribes.forEach(sub => sub());
-          } else {
-            await testRunner.runTests(
-              tests,
-              watcher,
-              onTestFileStart,
-              onResult,
-              onFailure,
-              testRunnerOptions,
-            );
+      const testsByRunner = this._partitionTests(testRunners, tests);
+
+      if (testsByRunner) {
+        try {
+          for (const runner of Object.keys(testRunners)) {
+            const testRunner = testRunners[runner];
+            const context = contextsByTestRunner.get(testRunner);
+
+            invariant(context);
+
+            const tests = testsByRunner[runner];
+
+            const testRunnerOptions = {
+              serial: runInBand || Boolean(testRunner.isSerial),
+            };
+
+            if (testRunner.supportsEventEmitters) {
+              const unsubscribes = [
+                testRunner.on('test-file-start', ([test]) =>
+                  onTestFileStart(test),
+                ),
+                testRunner.on('test-file-success', ([test, testResult]) =>
+                  onResult(test, testResult),
+                ),
+                testRunner.on('test-file-failure', ([test, error]) =>
+                  onFailure(test, error),
+                ),
+                testRunner.on(
+                  'test-case-result',
+                  ([testPath, testCaseResult]) => {
+                    const test: Test = {context, path: testPath};
+                    this._dispatcher.onTestCaseResult(test, testCaseResult);
+                  },
+                ),
+              ];
+
+              await testRunner.runTests(tests, watcher, testRunnerOptions);
+
+              unsubscribes.forEach(sub => sub());
+            } else {
+              await testRunner.runTests(
+                tests,
+                watcher,
+                onTestFileStart,
+                onResult,
+                onFailure,
+                testRunnerOptions,
+              );
+            }
+          }
+        } catch (error) {
+          if (!watcher.isInterrupted()) {
+            throw error;
           }
-        }
-      } catch (error) {
-        if (!watcher.isInterrupted()) {
-          throw error;
         }
       }
+    } catch (error) {
+      aggregatedResults.runExecError = buildExecError(error);
+      await this._dispatcher.onRunComplete(testContexts, aggregatedResults);
+      throw error;
     }
 
     await updateSnapshotState();

packages/jest-core/src/__tests__/TestScheduler.test.js

@@ -15,6 +15,7 @@ import {
   VerboseReporter,
 } from '@jest/reporters';
 import {makeGlobalConfig, makeProjectConfig} from '@jest/test-utils';
+import * as transform from '@jest/transform';
 import {createTestScheduler} from '../TestScheduler';
 import * as testSchedulerHelper from '../testSchedulerHelper';
 
@@ -28,8 +29,13 @@ jest
         onTestStart() {},
       })),
     {virtual: true},
-  );
-
+  )
+  .mock('@jest/transform', () => {
+    return {
+      __esModule: true,
+      ...jest.requireActual('@jest/transform'),
+    };
+  });
 const mockSerialRunner = {
   isSerial: true,
   runTests: jest.fn(),
@@ -233,6 +239,134 @@ test('.addReporter() .removeReporter()', async () => {
   expect(scheduler._dispatcher._reporters).not.toContain(reporter);
 });
 
+describe('scheduleTests should always dispatch runStart and runComplete events', () => {
+  const mockReporter = {
+    onRunComplete: jest.fn(),
+    onRunStart: jest.fn(),
+  };
+
+  const errorMsg = 'runtime-error';
+  let scheduler, t;
+
+  beforeEach(async () => {
+    mockReporter.onRunStart.mockClear();
+    mockReporter.onRunComplete.mockClear();
+
+    t = {
+      context: {
+        config: makeProjectConfig({
+          moduleFileExtensions: ['.js'],
+          rootDir: './',
+          runner: 'jest-runner-serial',
+          transform: [],
+        }),
+        hasteFS: {
+          matchFiles: jest.fn(() => []),
+        },
+      },
+      path: './test/path.js',
+    };
+
+    scheduler = await createTestScheduler(makeGlobalConfig(), {}, {});
+    scheduler.addReporter(mockReporter);
+  });
+
+  test('during normal run', async () => {
+    expect.hasAssertions();
+    const result = await scheduler.scheduleTests([t], {
+      isInterrupted: jest.fn(),
+      isWatchMode: () => true,
+      setState: jest.fn(),
+    });
+
+    expect(result.numTotalTestSuites).toEqual(1);
+
+    expect(mockReporter.onRunStart).toBeCalledTimes(1);
+    expect(mockReporter.onRunComplete).toBeCalledTimes(1);
+    const aggregatedResult = mockReporter.onRunComplete.mock.calls[0][1];
+    expect(aggregatedResult.runExecError).toBeUndefined();
+
+    expect(aggregatedResult).toEqual(result);
+  });
+  test.each`
+    runtimeError                                  | message
+    ${errorMsg}                                   | ${errorMsg}
+    ${123}                                        | ${'123'}
+    ${new Error(errorMsg)}                        | ${errorMsg}
+    ${{message: errorMsg}}                        | ${errorMsg}
+    ${{message: errorMsg, stack: 'stack-string'}} | ${errorMsg}
+    ${`${errorMsg}\n Require stack:xxxx`}         | ${errorMsg}
+  `('with runtime error: $runtimeError', async ({runtimeError, message}) => {
+    expect.hasAssertions();
+
+    const spyCreateScriptTransformer = jest.spyOn(
+      transform,
+      'createScriptTransformer',
+    );
+    spyCreateScriptTransformer.mockImplementation(async () => {
+      throw runtimeError;
+    });
+
+    await expect(
+      scheduler.scheduleTests([t], {
+        isInterrupted: jest.fn(),
+        isWatchMode: () => true,
+        setState: jest.fn(),
+      }),
+    ).rejects.toEqual(runtimeError);
+
+    expect(mockReporter.onRunStart).toBeCalledTimes(1);
+    expect(mockReporter.onRunComplete).toBeCalledTimes(1);
+    const aggregatedResult = mockReporter.onRunComplete.mock.calls[0][1];
+    expect(aggregatedResult.runExecError.message).toEqual(message);
+    expect(aggregatedResult.runExecError.stack.length).toBeGreaterThan(0);
+
+    spyCreateScriptTransformer.mockRestore();
+  });
+  test.each`
+    watchMode | isInterrupted | hasExecError
+    ${false}  | ${false}      | ${true}
+    ${true}   | ${false}      | ${true}
+    ${true}   | ${true}       | ${false}
+  `(
+    'with runner exception: watchMode=$watchMode, isInterrupted=$isInterrupted',
+    async ({watchMode, isInterrupted, hasExecError}) => {
+      expect.hasAssertions();
+
+      mockSerialRunner.runTests.mockImplementation(() => {
+        throw errorMsg;
+      });
+
+      try {
+        const result = await scheduler.scheduleTests([t], {
+          isInterrupted: () => isInterrupted,
+          isWatchMode: () => watchMode,
+          setState: jest.fn(),
+        });
+        if (hasExecError) {
+          throw new Error('should throw exception');
+        }
+        expect(result.runExecError).toBeUndefined();
+      } catch (e) {
+        expect(e).toEqual(errorMsg);
+      }
+
+      expect(mockReporter.onRunStart).toBeCalledTimes(1);
+      expect(mockReporter.onRunComplete).toBeCalledTimes(1);
+
+      const aggregatedResult = mockReporter.onRunComplete.mock.calls[0][1];
+      if (hasExecError) {
+        expect(aggregatedResult.runExecError.message).toEqual(errorMsg);
+        expect(aggregatedResult.runExecError.stack.length).toBeGreaterThan(0);
+      } else {
+        expect(aggregatedResult.runExecError).toBeUndefined();
+      }
+
+      mockSerialRunner.runTests.mockReset();
+    },
+  );
+});
+
 test('schedule tests run in parallel per default', async () => {
   const scheduler = await createTestScheduler(makeGlobalConfig(), {}, {});
   const test = {

packages/jest-test-result/src/types.ts

@@ -68,6 +68,7 @@ export type AggregatedResultWithoutCoverage = {
   success: boolean;
   testResults: Array<TestResult>;
   wasInterrupted: boolean;
+  runExecError?: SerializableError;
 };
 
 export type AggregatedResult = AggregatedResultWithoutCoverage & {