Fix test retries that contain snapshots (#8629)

* Fix bug with snapshots and retries * Update CHANGELOG.md * Address e2e tests PR comments * Use correct type for snapshotState * Simplify implementation and decouple it from expect
jestjs · Jul 4, 2019 · dd3d7b0 · dd3d7b0
1 parent 2b64bb4
commit dd3d7b0
Show file tree

Hide file tree

Showing 7 changed files with 298 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,7 @@
 - `[jest-snapshot]` Prevent inline snapshots from drifting when inline snapshots are updated ([#8492](https://github.com/facebook/jest/pull/8492))
 - `[jest-haste-map]` Don't throw on missing mapper in Node crawler ([#8558](https://github.com/facebook/jest/pull/8558))
 - `[jest-core]` Fix incorrect `passWithNoTests` warning ([#8595](https://github.com/facebook/jest/pull/8595))
+- `[jest-snapshots]` Fix test retries that contain snapshots ([#8629](https://github.com/facebook/jest/pull/8629))
 
 ### Chore & Maintenance
 

diff --git a/e2e/__tests__/toMatchInlineSnapshotWithRetries.test.ts b/e2e/__tests__/toMatchInlineSnapshotWithRetries.test.ts
@@ -0,0 +1,141 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import path from 'path';
+import {cleanup, makeTemplate, writeFiles} from '../Utils';
+import runJest from '../runJest';
+
+const DIR = path.resolve(__dirname, '../to-match-inline-snapshot-with-retries');
+const TESTS_DIR = path.resolve(DIR, '__tests__');
+
+beforeEach(() => cleanup(TESTS_DIR));
+afterAll(() => cleanup(TESTS_DIR));
+
+test('works with a single snapshot', () => {
+  const filename = 'basic-support.test.js';
+  const template = makeTemplate(`
+    let index = 0;
+    afterEach(() => {
+      index += 1;
+    });
+    jest.retryTimes($2);
+    test('snapshots', () => expect($1).toMatchInlineSnapshot(\`3\`));
+  `);
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['3', '1' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, ['-w=1', '--ci=false', filename]);
+    expect(stderr).toMatch('Snapshots:   1 passed, 1 total');
+    expect(status).toBe(0);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '2' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Received: 2');
+    expect(stderr).toMatch('1 snapshot failed from 1 test suite.');
+    expect(status).toBe(1);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '4' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Snapshots:   1 passed, 1 total');
+    expect(status).toBe(0);
+  }
+});
+
+test('works when a different assertion is failing', () => {
+  const filename = 'basic-support.test.js';
+  const template = makeTemplate(`
+    jest.retryTimes($1);
+    test('snapshots', () => {
+      expect(3).toMatchInlineSnapshot(\`3\`);
+      expect(false).toBe(true);
+    });
+  `);
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['4']),
+    });
+    const {stderr, status} = runJest(DIR, ['-w=1', '--ci=false', filename]);
+    expect(stderr).toMatch('Test Suites: 1 failed, 1 total');
+    expect(stderr).toMatch('Snapshots:   1 passed, 1 total');
+    expect(status).toBe(1);
+  }
+});
+
+test('works when multiple tests have snapshots but only one of them failed multiple times', () => {
+  const filename = 'basic-support.test.js';
+  const template = makeTemplate(`
+    test('passing snapshots', () => expect(1).toMatchInlineSnapshot(\`1\`));
+    describe('with retries', () => {
+      let index = 0;
+      afterEach(() => {
+        index += 1;
+      });
+      jest.retryTimes($2);
+      test('snapshots', () => expect($1).toMatchInlineSnapshot(\`3\`));
+    });
+  `);
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['3', '2' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, ['-w=1', '--ci=false', filename]);
+    expect(stderr).toMatch('Snapshots:   2 passed, 2 total');
+    expect(status).toBe(0);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '2' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Snapshot name: `with retries snapshots 1`');
+    expect(stderr).toMatch('Received: 2');
+    expect(stderr).toMatch('1 snapshot failed from 1 test suite.');
+    expect(status).toBe(1);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '4' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Snapshots:   1 passed, 1 total');
+    expect(status).toBe(0);
+  }
+});
diff --git a/e2e/__tests__/toMatchSnapshotWithRetries.test.ts b/e2e/__tests__/toMatchSnapshotWithRetries.test.ts
@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import path from 'path';
+import {cleanup, makeTemplate, writeFiles} from '../Utils';
+import runJest from '../runJest';
+
+const DIR = path.resolve(__dirname, '../to-match-snapshot-with-retries');
+const TESTS_DIR = path.resolve(DIR, '__tests__');
+
+beforeEach(() => cleanup(TESTS_DIR));
+afterAll(() => cleanup(TESTS_DIR));
+
+test('works with a single snapshot', () => {
+  const filename = 'basic-support.test.js';
+  const template = makeTemplate(`
+    let index = 0;
+    afterEach(() => {
+      index += 1;
+    });
+    jest.retryTimes($2);
+    test('snapshots', () => expect($1).toMatchSnapshot());
+  `);
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['3', '1' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, ['-w=1', '--ci=false', filename]);
+    expect(stderr).toMatch('1 snapshot written from 1 test suite.');
+    expect(status).toBe(0);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '2' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Received: 2');
+    expect(stderr).toMatch('1 snapshot failed from 1 test suite.');
+    expect(status).toBe(1);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '4' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Snapshots:   1 passed, 1 total');
+    expect(status).toBe(0);
+  }
+});
+
+test('works when multiple tests have snapshots but only one of them failed multiple times', () => {
+  const filename = 'basic-support.test.js';
+  const template = makeTemplate(`
+    test('passing snapshots', () => expect('foo').toMatchSnapshot());
+    describe('with retries', () => {
+      let index = 0;
+      afterEach(() => {
+        index += 1;
+      });
+      jest.retryTimes($2);
+      test('snapshots', () => expect($1).toMatchSnapshot());
+    });
+  `);
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['3', '2' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, ['-w=1', '--ci=false', filename]);
+    expect(stderr).toMatch('2 snapshots written from 1 test suite.');
+    expect(status).toBe(0);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '2' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Received: 2');
+    expect(stderr).toMatch('1 snapshot failed from 1 test suite.');
+    expect(status).toBe(1);
+  }
+
+  {
+    writeFiles(TESTS_DIR, {
+      [filename]: template(['index', '4' /* retries */]),
+    });
+    const {stderr, status} = runJest(DIR, [
+      '-w=1',
+      '--ci=false',
+      '--testRunner=jest-circus/runner',
+      filename,
+    ]);
+    expect(stderr).toMatch('Snapshots:   1 passed, 1 total');
+    expect(status).toBe(0);
+  }
+});
diff --git a/e2e/to-match-inline-snapshot-with-retries/package.json b/e2e/to-match-inline-snapshot-with-retries/package.json
@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}
diff --git a/e2e/to-match-snapshot-with-retries/package.json b/e2e/to-match-snapshot-with-retries/package.json
@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}
diff --git a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts
@@ -12,6 +12,7 @@ import {extractExpectedAssertionsErrors, getState, setState} from 'expect';
 import {formatExecError, formatResultsErrors} from 'jest-message-util';
 import {
   SnapshotState,
+  SnapshotStateType,
   addSerializer,
   buildSnapshotResolver,
 } from 'jest-snapshot';
@@ -131,6 +132,8 @@ export const initialize = ({
   });
   setState({snapshotState, testPath});
 
+  addEventHandler(handleSnapshotStateAfterRetry(snapshotState));
+
   // Return it back to the outer scope (test runner outside the VM).
   return {globals, snapshotState};
 };
@@ -243,6 +246,17 @@ export const runAndTransformResultsToJestFormat = async ({
   };
 };
 
+const handleSnapshotStateAfterRetry = (snapshotState: SnapshotStateType) => (
+  event: Circus.Event,
+) => {
+  switch (event.name) {
+    case 'test_retry': {
+      // Clear any snapshot data that occurred in previous test run
+      snapshotState.clear();
+    }
+  }
+};
+
 const eventHandler = (event: Circus.Event) => {
   switch (event.name) {
     case 'test_start': {

diff --git a/packages/jest-snapshot/src/State.ts b/packages/jest-snapshot/src/State.ts
@@ -42,6 +42,7 @@ export default class SnapshotState {
   private _index: number;
   private _updateSnapshot: Config.SnapshotUpdateState;
   private _snapshotData: SnapshotData;
+  private _initialData: SnapshotData;
   private _snapshotPath: Config.Path;
   private _inlineSnapshots: Array<InlineSnapshot>;
   private _uncheckedKeys: Set<string>;
@@ -60,6 +61,7 @@ export default class SnapshotState {
       this._snapshotPath,
       options.updateSnapshot,
     );
+    this._initialData = data;
     this._snapshotData = data;
     this._dirty = dirty;
     this._getBabelTraverse = options.getBabelTraverse;
@@ -108,6 +110,17 @@ export default class SnapshotState {
     }
   }
 
+  clear() {
+    this._snapshotData = this._initialData;
+    this._inlineSnapshots = [];
+    this._counters = new Map();
+    this._index = 0;
+    this.added = 0;
+    this.matched = 0;
+    this.unmatched = 0;
+    this.updated = 0;
+  }
+
   save() {
     const hasExternalSnapshots = Object.keys(this._snapshotData).length;
     const hasInlineSnapshots = this._inlineSnapshots.length;