test: organize golden files under __snapshots__ folder (microsoft#3494)

.github/workflows/tests.yml

@@ -46,8 +46,8 @@ jobs:
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
-        name: ${{ matrix.browser }}-${{ matrix.os }}-output
-        path: test/output-${{ matrix.browser }}
+        name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
+        path: test-results
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -77,8 +77,8 @@ jobs:
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
-        name: ${{ matrix.browser }}-mac-output
-        path: test/output-${{ matrix.browser }}
+        name: ${{ matrix.browser }}-mac-test-results
+        path: test-results
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -112,8 +112,8 @@ jobs:
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
-        name: ${{ matrix.browser }}-win-output
-        path: test/output-${{ matrix.browser }}
+        name: ${{ matrix.browser }}-win-test-results
+        path: test-results
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -169,8 +169,8 @@ jobs:
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
-        name: headful-${{ matrix.browser }}-linux-output
-        path: test/output-${{ matrix.browser }}
+        name: headful-${{ matrix.browser }}-linux-test-results
+        path: test-results
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
@@ -207,8 +207,8 @@ jobs:
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
-        name: wire-${{ matrix.browser }}-linux-output
-        path: test/output-${{ matrix.browser }}
+        name: wire-${{ matrix.browser }}-linux-test-results
+        path: test-results
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:

.gitignore

@@ -1,7 +1,6 @@
 /node_modules/
-/test/output-chromium
-/test/output-firefox
-/test/output-webkit
+/test-results/
+/test/coverage-report
 /test/test-user-data-dir*
 .local-browsers/
 /.dev_profile*

test/base.fixture.ts

@@ -26,6 +26,7 @@ import { setUnderTest } from '../lib/helper';
 import { installCoverageHooks } from './runner/coverage';
 import { valueFromEnv } from './runner/utils';
 import { registerFixture, registerWorkerFixture} from './runner/fixtures';
+import './runner/builtin.fixtures';
 
 import {mkdtempAsync, removeFolderAsync} from './utils';
 
@@ -37,13 +38,11 @@ const platform = os.platform();
 declare global {
   interface WorkerState {
     asset: (path: string) => string;
-    parallelIndex: number;
     defaultBrowserOptions: LaunchOptions;
     golden: (path: string) => string;
     playwright: typeof import('../index');
     browserType: BrowserType<Browser>;
     browser: Browser;
-    tmpDir: string;
   }
   interface FixtureState {
     toImpl: (rpcObject: any) => any;
@@ -62,10 +61,6 @@ declare global {
 (global as any).FFOX = browserName === 'firefox';
 (global as any).WEBKIT = browserName === 'webkit';
 
-registerWorkerFixture('parallelIndex', async ({}, test) => {
-  await test(parseInt(process.env.JEST_WORKER_ID, 10) - 1);
-});
-
 registerWorkerFixture('httpService', async ({parallelIndex}, test) => {
   const assetsPath = path.join(__dirname, 'assets');
   const cachedPath = path.join(__dirname, 'assets', 'cached');
@@ -139,7 +134,7 @@ registerWorkerFixture('playwright', async({parallelIndex}, test) => {
 
   async function teardownCoverage() {
     uninstall();
-    const coveragePath = path.join(path.join(__dirname, 'output-' + browserName), 'coverage', parallelIndex + '.json');
+    const coveragePath = path.join(path.join(__dirname, 'coverage-report'), 'coverage', parallelIndex + '.json');
     const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key));
     await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
     await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
@@ -205,5 +200,5 @@ registerWorkerFixture('asset', async ({}, test) => {
 });
 
 registerWorkerFixture('golden', async ({browserName}, test) => {
-  await test(p => path.join(__dirname, `golden-${browserName}`, p));
+  await test(p => path.join(`${browserName}`, p));
 });

test/chromium/oopif.spec.ts

@@ -155,7 +155,7 @@ it.skip(!CHROMIUM)('should take screenshot', async({browser, page, server, golde
   await page.goto(server.PREFIX + '/dynamic-oopif.html');
   expect(page.frames().length).toBe(2);
   expect(await countOOPIFs(browser)).toBe(1);
-  expect(await page.screenshot()).toMatchImage(golden('screenshot-oopif.png'));
+  expect(await page.screenshot()).toMatchImage(golden('screenshot-oopif.png'), { threshold: 0.3 });
 });
 
 it.skip(!CHROMIUM)('should load oopif iframes with subresources and request interception', async function({browser, page, server, context}) {

test/chromium/tracing.spec.ts

@@ -13,6 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import '../runner/builtin.fixtures';
 import '../base.fixture';
 import { registerFixture } from '../runner/fixtures';
 

test/defaultbrowsercontext.spec.ts

@@ -14,6 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import './runner/builtin.fixtures';
 import './base.fixture';
 import { registerFixture } from './runner/fixtures';
 import fs from 'fs';

test/download.spec.ts

@@ -13,6 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import './runner/builtin.fixtures';
 import './base.fixture';
 
 import fs from 'fs';

test/downloads-path.spec.ts

@@ -13,7 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+
+import './runner/builtin.fixtures';
 import './base.fixture';
+
 import { registerFixture } from './runner/fixtures';
 
 import path from 'path';

test/headful.spec.ts

@@ -13,6 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import './runner/builtin.fixtures';
 import './base.fixture';
 
 import utils from './utils';

test/page-screenshot.spec.ts

@@ -202,7 +202,7 @@ it.skip(ffheadful)('should work for canvas', async({page, server, golden}) => {
   await page.setViewportSize({width: 500, height: 500});
   await page.goto(server.PREFIX + '/screenshots/canvas.html');
   const screenshot = await page.screenshot();
-  expect(screenshot).toMatchImage(golden('screenshot-canvas.png'));
+  expect(screenshot).toMatchImage(golden('screenshot-canvas.png'), { threshold: 0.3 });
 });
 
 it.skip(ffheadful)('should work for translateZ', async({page, server, golden}) => {

test/pdf.spec.ts

@@ -13,6 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import './runner/builtin.fixtures';
 import './base.fixture';
 
 import fs from 'fs'

test/proxy.spec.ts

@@ -13,6 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import './runner/builtin.fixtures';
 import './base.fixture';
 
 import socks from 'socksv5';

test/runner/GoldenUtils.js

@@ -44,7 +44,7 @@ const GoldenComparators = {
  * @param {!string} mimeType
  * @return {?{diff: (!Object:undefined), errorMessage: (string|undefined)}}
  */
-function compareImages(actualBuffer, expectedBuffer, mimeType) {
+function compareImages(actualBuffer, expectedBuffer, mimeType, config = {}) {
   if (!actualBuffer || !(actualBuffer instanceof Buffer))
     return { errorMessage: 'Actual result should be Buffer.' };
 
@@ -56,7 +56,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType) {
     };
   }
   const diff = new PNG({width: expected.width, height: expected.height});
-  const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {threshold: 0.2});
+  const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {threshold: 0.2, ...config});
   return count > 0 ? { diff: PNG.sync.write(diff) } : null;
 }
 
@@ -88,8 +88,16 @@ function compareText(actual, expectedBuffer) {
  * @param {string} path
  * @return {!{pass: boolean, message: (undefined|string)}}
  */
-function compare(actual, expectedPath) {
+function compare(actual, name, options) {
+  const { relativeTestFile, snapshotDir, outputDir, updateSnapshots } = options;
+  let expectedPath;
+  const testAssetsDir = relativeTestFile.replace(/\.spec\.[jt]s/, '');
+  if (path.isAbsolute(name))
+    expectedPath = name;
+  else
+    expectedPath = path.join(snapshotDir, testAssetsDir, name);
   if (!fs.existsSync(expectedPath)) {
+    fs.mkdirSync(path.dirname(expectedPath), { recursive: true });
     fs.writeFileSync(expectedPath, actual);
     return {
       pass: false,
@@ -107,15 +115,35 @@ function compare(actual, expectedPath) {
     };
   }
 
-  const result = comparator(actual, expected, mimeType);
+  const result = comparator(actual, expected, mimeType, options.config);
   if (!result)
     return { pass: true };
 
-  const actualPath = addSuffix(expectedPath, '-actual');
-  const diffPath = addSuffix(expectedPath, '-diff', result.diffExtension);
+  if (updateSnapshots) {
+    fs.mkdirSync(path.dirname(expectedPath), { recursive: true });
+    fs.writeFileSync(expectedPath, actual);
+    return {
+      pass: true,
+      message: expectedPath + ' running with --update-snapshots, writing actual.'
+    };
+  }
+
+  let actualPath;
+  let diffPath;
+  if (path.isAbsolute(name)) {
+    actualPath = addSuffix(expectedPath, '-actual');
+    diffPath = addSuffix(expectedPath, '-diff', result.diffExtension);
+  } else {
+    const outputPath = path.join(outputDir, testAssetsDir, name);
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    const expectedPathOut = addSuffix(outputPath, '-expected');
+    actualPath = addSuffix(outputPath, '-actual');
+    diffPath = addSuffix(outputPath, '-diff', result.diffExtension);
+    fs.writeFileSync(expectedPathOut, expected);
+  }
   fs.writeFileSync(actualPath, actual);
   if (result.diff)
-    fs.writeFileSync(diffPath, result.diff);
+    fs.writeFileSync(diffPath, result.diff);  
 
   const output = [
     c.red(`Image comparison failed:`),

test/runner/builtin.fixtures.js

@@ -0,0 +1,48 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const util = require('util');
+const { mkdtempAsync, removeFolderAsync } = require('../utils');
+const { registerFixture, registerWorkerFixture } = require('./fixtures');
+const mkdirAsync = util.promisify(fs.mkdir.bind(fs));
+
+let workerId;
+let outputDir;
+
+registerWorkerFixture('parallelIndex', async ({}, test) => {
+  await test(workerId);
+});
+
+registerFixture('tmpDir', async ({}, test) => {
+  const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
+  await test(tmpDir);
+  await removeFolderAsync(tmpDir).catch(e => {});
+});
+
+registerWorkerFixture('outputDir', async ({}, test) => {
+  await mkdirAsync(outputDir, { recursive: true });
+  await test(outputDir);
+});
+
+function initializeWorker(options) {
+  workerId = options.workerId;
+  outputDir = options.outputDir;
+} 
+
+module.exports = { initializeWorker };

test/runner/checkCoverage.js

@@ -38,7 +38,7 @@ if (browserName !== 'chromium') {
 if (browserName === 'webkit')
   api.delete('browserContext.clearPermissions');
 
-const coverageDir = path.join(__dirname, '..', 'output-' + browserName, 'coverage');
+const coverageDir = path.join(__dirname, '..', 'coverage-report', 'coverage');
 
 const coveredMethods = new Set();
 for (const file of getCoverageFiles(coverageDir)) {

test/runner/fixturesUI.js

@@ -23,7 +23,6 @@ Error.stackTraceLimit = 15;
 global.testOptions = require('./testOptions');
 global.registerFixture = registerFixture;
 global.registerWorkerFixture = registerWorkerFixture;
-process.env.JEST_WORKER_ID = 1;
 
 const fixturePool = new FixturePool();
 let revertBabelRequire;