cherry-pick(#34556): fix(toMatchAriaSnapshot): fail test run when upd…

…ating missing snapshot (#34561)
microsoft · Jan 30, 2025 · 159210d · 159210d
1 parent fbad9f7
commit 159210d
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 8 deletions.
diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts
@@ -329,6 +329,8 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
         const jestError = isJestError(e) ? e : null;
         const error = jestError ? new ExpectError(jestError, customMessage, stackFrames) : e;
         if (jestError?.matcherResult.suggestedRebaseline) {
+          // NOTE: this is a workaround for the fact that we can't pass the suggested rebaseline
+          // for passing matchers. See toMatchAriaSnapshot for a counterpart.
           step.complete({ suggestedRebaseline: jestError?.matcherResult.suggestedRebaseline });
           return;
         }

diff --git a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts
@@ -140,6 +140,14 @@ export async function toMatchAriaSnapshot(
         return { pass: true, message: () => '', name: 'toMatchAriaSnapshot' };
       } else {
         const suggestedRebaseline = `\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent}  '))}\n{indent}\``;
+        if (updateSnapshots === 'missing') {
+          const message = 'A snapshot is not provided, generating new baseline.';
+          testInfo._hasNonRetriableError = true;
+          testInfo._failWithError(new Error(message));
+        }
+        // TODO: ideally, we should return "pass: true" here because this matcher passes
+        // when regenerating baselines. However, we can only access suggestedRebaseline in case
+        // of an error, so we fail here and workaround it in the expect implementation.
         return { pass: false, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline };
       }
     }

diff --git a/tests/playwright-test/update-aria-snapshot.spec.ts b/tests/playwright-test/update-aria-snapshot.spec.ts
@@ -84,7 +84,8 @@ test('should update missing snapshots', async ({ runInlineTest }, testInfo) => {
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Error: A snapshot is not provided, generating new baseline.');
 
   expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for:
 
@@ -129,7 +130,7 @@ test('should update multiple missing snapshots', async ({ runInlineTest }, testI
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
 
   expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for:
 
@@ -188,7 +189,7 @@ test('should generate baseline with regex', async ({ runInlineTest }, testInfo)
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
   const patchPath = testInfo.outputPath('test-results/rebaselines.patch');
   const data = fs.readFileSync(patchPath, 'utf-8');
   expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts
@@ -249,7 +250,7 @@ test('should generate baseline with special characters', async ({ runInlineTest
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
   const patchPath = testInfo.outputPath('test-results/rebaselines.patch');
   const data = fs.readFileSync(patchPath, 'utf-8');
   expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts
@@ -314,7 +315,7 @@ test('should update missing snapshots in tsx', async ({ runInlineTest }, testInf
     `,
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
   const patchPath = testInfo.outputPath('test-results/rebaselines.patch');
   const data = fs.readFileSync(patchPath, 'utf-8');
   expect(trimPatch(data)).toBe(`diff --git a/src/button.test.tsx b/src/button.test.tsx
@@ -370,7 +371,7 @@ test('should update multiple files', async ({ runInlineTest }, testInfo) => {
     `,
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
 
   expect(stripAnsi(result.output).replace(/\\/g, '/')).toContain(`New baselines created for:
 
@@ -430,7 +431,7 @@ test('should generate baseline for input values', async ({ runInlineTest }, test
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
   const patchPath = testInfo.outputPath('test-results/rebaselines.patch');
   const data = fs.readFileSync(patchPath, 'utf-8');
   expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts
@@ -470,7 +471,7 @@ test('should update when options are specified', async ({ runInlineTest }, testI
     `
   });
 
-  expect(result.exitCode).toBe(0);
+  expect(result.exitCode).toBe(1);
   const patchPath = testInfo.outputPath('test-results/rebaselines.patch');
   const data = fs.readFileSync(patchPath, 'utf-8');
   expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts