diff --git a/.github/workflows/auto_roll.yml b/.github/workflows/auto_roll.yml index 021da9758db3f..40063c5b07f36 100644 --- a/.github/workflows/auto_roll.yml +++ b/.github/workflows/auto_roll.yml @@ -27,7 +27,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --retries=3 --timeout=30000 --global-timeout=5400000 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --retries=3 --timeout=30000 --global-timeout=5400000 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 81b4c2b561e1f..cb488910bef49 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json && npm run coverage" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json && npm run coverage" env: BROWSER: ${{ matrix.browser }} PWRUNNER_JSON_REPORT: "test-results/report.json" @@ -62,7 +62,7 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json + - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json env: BROWSER: ${{ matrix.browser }} PWRUNNER_JSON_REPORT: "test-results/report.json" @@ -90,7 +90,7 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json + - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json shell: bash env: BROWSER: ${{ matrix.browser }} @@ -141,7 +141,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" if: ${{ always() }} env: BROWSER: ${{ matrix.browser }} @@ -174,7 +174,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} PWWIRE: true @@ -206,7 +206,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} TRACING: true diff --git a/.gitignore b/.gitignore index b160baf88f0f5..2289dac322220 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /node_modules/ /test-results/ -/test-runner/test/test-results/ /test/coverage-report /test/test-user-data-dir* .local-browsers/ diff --git a/package-lock.json b/package-lock.json index cf2381082483f..07d28e6a8c887 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1172,6 +1172,20 @@ "fastq": "^1.6.0" } }, + "@playwright/test-runner": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.2.1.tgz", + "integrity": "sha512-ZZILhZZSo4Gv2T70HCkHRIUao3vHZpdSjVbjrXUWGaulxzoF77i0omRth7Kie+qBWhOIlpZe1rgBDQW801GTkw==", + "dev": true, + "requires": { + "colors": "^1.4.0", + "expect": "^26.4.2", + "jpeg-js": "^0.4.2", + "pixelmatch": "^5.2.1", + "pngjs": "^5.0.0", + "text-diff": "^1.0.1" + } + }, "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", diff --git a/package.json b/package.json index 1aa428c40c1f6..db3955c54576f 100644 --- a/package.json +++ b/package.json @@ -9,19 +9,19 @@ "node": ">=10.15.0" }, "scripts": { - "ctest": "cross-env BROWSER=chromium node test-runner/cli test/", - "ftest": "cross-env BROWSER=firefox node test-runner/cli test/", - "wtest": "cross-env BROWSER=webkit node test-runner/cli test/", - "test": "node test-runner/cli test/", + "ctest": "cross-env BROWSER=chromium test-runner test/", + "ftest": "cross-env BROWSER=firefox test-runner test/", + "wtest": "cross-env BROWSER=webkit test-runner test/", + "test": "test-runner test/", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .", "tsc": "tsc -p .", "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", - "test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js && npm run test-testrunner", + "test-infra": "test-runner utils/doclint/check_public_api/test/test.js && test-runner utils/doclint/preprocessor/test.js", "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra", "clean": "rimraf lib && rimraf types", "prepare": "node install-from-github.js", - "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run build-testrunner && npm run generate-types", + "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types", "watch": "node utils/watch.js", "test-types": "npm run generate-types && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && npm run typecheck-tests", "generate-types": "node utils/generate_types/", @@ -30,9 +30,7 @@ "roll-browser": "node utils/roll_browser.js", "coverage": "node test/checkCoverage.js", "check-deps": "node utils/check_deps.js", - "show-trace": "node utils/showTestTraces.js", - "build-testrunner": "tsc -p test-runner", - "test-testrunner": "node test-runner/cli test-runner/test" + "show-trace": "node utils/showTestTraces.js" }, "author": { "name": "Microsoft Corporation" @@ -55,6 +53,7 @@ "@babel/core": "^7.11.4", "@babel/preset-env": "^7.11.0", "@babel/preset-typescript": "^7.10.4", + "@playwright/test-runner": "^0.2.1", "@types/babel__core": "^7.1.9", "@types/debug": "^4.1.5", "@types/extract-zip": "^1.6.2", diff --git a/test-runner/cli.js b/test-runner/cli.js deleted file mode 100644 index 822d1f86c8480..0000000000000 --- a/test-runner/cli.js +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env node -module.exports = require('./lib/cli') \ No newline at end of file diff --git a/test-runner/package.json b/test-runner/package.json deleted file mode 100644 index 6631bf9378c62..0000000000000 --- a/test-runner/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "test-runner", - "version": "0.0.7", - "bin": { - "test-runner": "./cli.js" - }, - "main": "./lib/index.js" -} \ No newline at end of file diff --git a/test-runner/src/builtin.fixtures.ts b/test-runner/src/builtin.fixtures.ts deleted file mode 100644 index 9343507620197..0000000000000 --- a/test-runner/src/builtin.fixtures.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import os from 'os'; -import path from 'path'; -import { promisify } from 'util'; -import fs from 'fs'; -import rimraf from 'rimraf'; -import { registerFixture } from './fixtures'; -import { Test, Suite } from './test'; - -interface DescribeFunction { - describe(name: string, inner: () => void): void; - describe(name: string, modifier: (suite: Suite) => any, inner: () => void): void; -} - -interface ItFunction { - it(name: string, inner: (state: STATE) => Promise | void): void; - it(name: string, modifier: (test: Test) => any, inner: (state: STATE) => Promise | void): void; -} - -declare global { - const describe: DescribeFunction['describe']; - const fdescribe: DescribeFunction['describe']; - const xdescribe: DescribeFunction['describe']; - - const it: ItFunction['it']; - const fit: ItFunction['it']; - const xit: ItFunction['it']; - - const beforeEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise) => void; - const afterEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise) => void; - const beforeAll: (inner: (state: WorkerState & FixtureParameters) => Promise) => void; - const afterAll: (inner: (state: WorkerState & FixtureParameters) => Promise) => void; -} - -const mkdtempAsync = promisify(fs.mkdtemp); -const removeFolderAsync = promisify(rimraf); - -declare global { - interface FixtureParameters { - parallelIndex: number; - } - interface TestState { - tmpDir: string; - } -} - -export {parameters, registerFixture, registerWorkerFixture, registerParameter} from './fixtures'; - -registerFixture('tmpDir', async ({}, test) => { - const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-')); - await test(tmpDir); - await removeFolderAsync(tmpDir).catch(e => {}); -}); diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts deleted file mode 100644 index c07e089bfbe0d..0000000000000 --- a/test-runner/src/cli.ts +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import program from 'commander'; -import * as fs from 'fs'; -import * as path from 'path'; -import { run, RunnerConfig } from '.'; -import PytestReporter from './reporters/pytest'; -import DotReporter from './reporters/dot'; -import ListReporter from './reporters/list'; -import JSONReporter from './reporters/json'; -import { Reporter } from './reporter'; -import { Multiplexer } from './reporters/multiplexer'; - -export const reporters = { - 'dot': DotReporter, - 'list': ListReporter, - 'json': JSONReporter, - 'pytest': PytestReporter, -}; - -program - .version('Version ' + /** @type {any} */ (require)('../package.json').version) - .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) - .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') - .option('--global-timeout ', 'Specify maximum time this test suite can run (in milliseconds), default: 0 for unlimited', '0') - .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', String(Math.ceil(require('os').cpus().length / 2))) - .option('--reporter ', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot') - .option('--repeat-each ', 'Specify how many times to run the tests', '1') - .option('--retries ', 'Specify retry count', '0') - .option('--trial-run', 'Only collect the matching tests and report them as passing') - .option('--quiet', 'Suppress stdio', false) - .option('--debug', 'Run tests in-process for debugging', false) - .option('--output ', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results')) - .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000') - .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') - .action(async command => { - const testDir = path.resolve(process.cwd(), command.args[0]); - const config: RunnerConfig = { - debug: command.debug, - forbidOnly: command.forbidOnly, - quiet: command.quiet, - grep: command.grep, - jobs: parseInt(command.jobs, 10), - outputDir: command.output, - repeatEach: parseInt(command.repeatEach, 10), - retries: parseInt(command.retries, 10), - snapshotDir: path.join(testDir, '__snapshots__'), - testDir, - timeout: parseInt(command.timeout, 10), - globalTimeout: parseInt(command.globalTimeout, 10), - trialRun: command.trialRun, - updateSnapshots: command.updateSnapshots - }; - - const reporterList = command.reporter.split(','); - const reporterObjects: Reporter[] = reporterList.map(c => { - if (reporters[c]) - return new reporters[c](); - try { - const p = path.resolve(process.cwd(), c); - return new (require(p).default)(); - } catch (e) { - console.error('Invalid reporter ' + c, e); - process.exit(1); - } - }); - - const files = collectFiles(testDir, '', command.args.slice(1)); - const result = await run(config, files, new Multiplexer(reporterObjects)); - if (result === 'forbid-only') { - console.error('====================================='); - console.error(' --forbid-only found a focused test.'); - console.error('====================================='); - process.exit(1); - } - - if (result === 'no-tests') { - console.error('================='); - console.error(' no tests found.'); - console.error('================='); - process.exit(1); - } - - process.exit(result === 'failed' ? 1 : 0); - }); - -program.parse(process.argv); - -function collectFiles(testDir: string, dir: string, filters: string[]): string[] { - const fullDir = path.join(testDir, dir); - if (fs.statSync(fullDir).isFile()) - return [fullDir]; - const files = []; - for (const name of fs.readdirSync(fullDir)) { - if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) { - files.push(...collectFiles(testDir, path.join(dir, name), filters)); - continue; - } - if (!name.endsWith('spec.ts')) - continue; - const relativeName = path.join(dir, name); - const fullName = path.join(testDir, relativeName); - if (!filters.length) { - files.push(fullName); - continue; - } - for (const filter of filters) { - if (relativeName.includes(filter)) { - files.push(fullName); - break; - } - } - } - return files; -} diff --git a/test-runner/src/expect.ts b/test-runner/src/expect.ts deleted file mode 100644 index 5bed3db694e50..0000000000000 --- a/test-runner/src/expect.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { compare } from './golden'; -import { RunnerConfig } from './runnerConfig'; - -declare global { - const expect: typeof import('expect'); -} - -declare module 'expect/build/types' { - interface Matchers { - toMatchImage(path: string, options?: { threshold?: number }): R; - } -} - -global['expect'] = require('expect'); - -let testFile: string; - -export function initializeImageMatcher(config: RunnerConfig) { - function toMatchImage(received: Buffer, name: string, options?: { threshold?: number }) { - const { pass, message } = compare(received, name, config, testFile, options); - return { pass, message: () => message }; - } - expect.extend({ toMatchImage }); -} - -export function setCurrentTestFile(file: string) { - testFile = file; -} \ No newline at end of file diff --git a/test-runner/src/fixtures.ts b/test-runner/src/fixtures.ts deleted file mode 100644 index 52918d09d077b..0000000000000 --- a/test-runner/src/fixtures.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import debug from 'debug'; -import { RunnerConfig } from './runnerConfig'; -import { serializeError, Test, TestResult } from './test'; -import { raceAgainstTimeout } from './util'; - -type Scope = 'test' | 'worker'; - -type FixtureRegistration = { - name: string; - scope: Scope; - fn: Function; -}; - -export type TestInfo = { - config: RunnerConfig; - test: Test; - result: TestResult; -}; - -const registrations = new Map(); -const registrationsByFile = new Map(); -export let parameters: any = {}; -export const parameterRegistrations = new Map(); - -export function setParameters(params: any) { - parameters = Object.assign(parameters, params); - for (const name of Object.keys(params)) - registerWorkerFixture(name, async ({}, test) => await test(parameters[name])); -} - -class Fixture { - pool: FixturePool; - name: string; - scope: Scope; - fn: Function; - deps: string[]; - usages: Set; - hasGeneratorValue: boolean; - value: any; - _teardownFenceCallback: (value?: unknown) => void; - _tearDownComplete: Promise; - _setup = false; - _teardown = false; - - constructor(pool: FixturePool, name: string, scope: Scope, fn: any) { - this.pool = pool; - this.name = name; - this.scope = scope; - this.fn = fn; - this.deps = fixtureParameterNames(this.fn); - this.usages = new Set(); - this.hasGeneratorValue = name in parameters; - this.value = this.hasGeneratorValue ? parameters[name] : null; - } - - async setup(config: RunnerConfig, info?: TestInfo) { - if (this.hasGeneratorValue) - return; - for (const name of this.deps) { - await this.pool.setupFixture(name, config, info); - this.pool.instances.get(name).usages.add(this.name); - } - - const params = {}; - for (const n of this.deps) - params[n] = this.pool.instances.get(n).value; - let setupFenceFulfill: { (): void; (value?: unknown): void; }; - let setupFenceReject: { (arg0: any): any; (reason?: any): void; }; - const setupFence = new Promise((f, r) => { setupFenceFulfill = f; setupFenceReject = r; }); - const teardownFence = new Promise(f => this._teardownFenceCallback = f); - debug('pw:test:hook')(`setup "${this.name}"`); - const param = info || config; - this._tearDownComplete = this.fn(params, async (value: any) => { - this.value = value; - setupFenceFulfill(); - return await teardownFence; - }, param).catch((e: any) => setupFenceReject(e)); - await setupFence; - this._setup = true; - } - - async teardown() { - if (this.hasGeneratorValue) - return; - if (this._teardown) - return; - this._teardown = true; - for (const name of this.usages) { - const fixture = this.pool.instances.get(name); - if (!fixture) - continue; - await fixture.teardown(); - } - if (this._setup) { - debug('pw:test:hook')(`teardown "${this.name}"`); - this._teardownFenceCallback(); - await this._tearDownComplete; - } - this.pool.instances.delete(this.name); - } -} - -export class FixturePool { - instances: Map; - constructor() { - this.instances = new Map(); - } - - async setupFixture(name: string, config: RunnerConfig, info?: TestInfo) { - let fixture = this.instances.get(name); - if (fixture) - return fixture; - - if (!registrations.has(name)) - throw new Error('Unknown fixture: ' + name); - const { scope, fn } = registrations.get(name); - fixture = new Fixture(this, name, scope, fn); - this.instances.set(name, fixture); - await fixture.setup(config, info); - return fixture; - } - - async teardownScope(scope: string) { - for (const [, fixture] of this.instances) { - if (fixture.scope === scope) - await fixture.teardown(); - } - } - - async resolveParametersAndRun(fn: Function, config: RunnerConfig, info?: TestInfo) { - const names = fixtureParameterNames(fn); - for (const name of names) - await this.setupFixture(name, config, info); - const params = {}; - for (const n of names) - params[n] = this.instances.get(n).value; - return fn(params); - } - - async runTestWithFixturesAndTimeout(fn: Function, timeout: number, info: TestInfo) { - const { timedOut } = await raceAgainstTimeout(this._runTestWithFixtures(fn, info), timeout); - // Do not overwrite test failure upon timeout in fixture. - if (timedOut && info.result.status === 'passed') - info.result.status = 'timedOut'; - } - - async _runTestWithFixtures(fn: Function, info: TestInfo) { - try { - await this.resolveParametersAndRun(fn, info.config, info); - info.result.status = 'passed'; - } catch (error) { - // Prefer original error to the fixture teardown error or timeout. - if (info.result.status === 'passed') { - info.result.status = 'failed'; - info.result.error = serializeError(error); - } - } - try { - await this.teardownScope('test'); - } catch (error) { - // Prefer original error to the fixture teardown error or timeout. - if (info.result.status === 'passed') { - info.result.status = 'failed'; - info.result.error = serializeError(error); - } - } - } -} - -export function fixturesForCallback(callback: Function): string[] { - const names = new Set(); - const visit = (callback: Function) => { - for (const name of fixtureParameterNames(callback)) { - if (name in names) - continue; - names.add(name); - if (!registrations.has(name)) - throw new Error('Using undefined fixture ' + name); - - const { fn } = registrations.get(name); - visit(fn); - } - }; - visit(callback); - const result = [...names]; - result.sort(); - return result; -} - -function fixtureParameterNames(fn: Function): string[] { - const text = fn.toString(); - const match = text.match(/async(?:\s+function)?\s*\(\s*{\s*([^}]*)\s*}/); - if (!match || !match[1].trim()) - return []; - const signature = match[1]; - return signature.split(',').map((t: string) => t.trim()); -} - -function innerRegisterFixture(name: string, scope: Scope, fn: Function, caller: Function) { - const obj = {stack: ''}; - Error.captureStackTrace(obj, caller); - const stackFrame = obj.stack.split('\n')[2]; - const location = stackFrame.replace(/.*at Object. \((.*)\)/, '$1'); - const file = location.replace(/^(.+):\d+:\d+$/, '$1'); - const registration = { name, scope, fn, file, location }; - registrations.set(name, registration); - if (!registrationsByFile.has(file)) - registrationsByFile.set(file, []); - registrationsByFile.get(file).push(registration); -} - -export function registerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise, info: TestInfo) => Promise) { - innerRegisterFixture(name, 'test', fn, registerFixture); -} - -export function registerWorkerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise, config: RunnerConfig) => Promise) { - innerRegisterFixture(name, 'worker', fn, registerWorkerFixture); -} - -export function registerParameter(name: string, fn: () => any) { - registerWorkerFixture(name, async ({}: any, test: Function) => await test(parameters[name])); - parameterRegistrations.set(name, fn); -} - -function collectRequires(file: string, result: Set) { - if (result.has(file)) - return; - result.add(file); - const cache = require.cache[file]; - if (!cache) - return; - const deps = cache.children.map((m: { id: any; }) => m.id).slice().reverse(); - for (const dep of deps) - collectRequires(dep, result); -} - -export function lookupRegistrations(file: string, scope: Scope) { - const deps = new Set(); - collectRequires(file, deps); - const allDeps = [...deps].reverse(); - const result = new Map(); - for (const dep of allDeps) { - const registrationList = registrationsByFile.get(dep); - if (!registrationList) - continue; - for (const r of registrationList) { - if (scope && r.scope !== scope) - continue; - result.set(r.name, r); - } - } - return result; -} - -export function rerunRegistrations(file: string, scope: Scope) { - // When we are running several tests in the same worker, we should re-run registrations before - // each file. That way we erase potential fixture overrides from the previous test runs. - for (const registration of lookupRegistrations(file, scope).values()) - registrations.set(registration.name, registration); -} diff --git a/test-runner/src/golden.ts b/test-runner/src/golden.ts deleted file mode 100644 index ba4c70b69077a..0000000000000 --- a/test-runner/src/golden.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import c from 'colors/safe'; -import fs from 'fs'; -import jpeg from 'jpeg-js'; -import path from 'path'; -import pixelmatch from 'pixelmatch'; -import { PNG } from 'pngjs'; -import Diff from 'text-diff'; -import { RunnerConfig } from './runnerConfig'; - -const extensionToMimeType = { - 'png': 'image/png', - 'txt': 'text/plain', - 'jpg': 'image/jpeg', - 'jpeg': 'image/jpeg', -}; - -const GoldenComparators = { - 'image/png': compareImages, - 'image/jpeg': compareImages, - 'text/plain': compareText -}; - -function compareImages(actualBuffer: Buffer, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: object; errorMessage?: string; } | null { - if (!actualBuffer || !(actualBuffer instanceof Buffer)) - return { errorMessage: 'Actual result should be Buffer.' }; - - const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpeg.decode(actualBuffer); - const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer); - if (expected.width !== actual.width || expected.height !== actual.height) { - return { - errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. ` - }; - } - const diff = new PNG({width: expected.width, height: expected.height}); - const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options }); - return count > 0 ? { diff: PNG.sync.write(diff) } : null; -} - -function compareText(actual: Buffer, expectedBuffer: Buffer): { diff?: object; errorMessage?: string; diffExtension?: string; } | null { - if (typeof actual !== 'string') - return { errorMessage: 'Actual result should be string' }; - const expected = expectedBuffer.toString('utf-8'); - if (expected === actual) - return null; - const diff = new Diff(); - const result = diff.main(expected, actual); - diff.cleanupSemantic(result); - let html = diff.prettyHtml(result); - const diffStylePath = path.join(__dirname, 'diffstyle.css'); - html = `` + html; - return { - diff: html, - diffExtension: '.html' - }; -} - -export function compare(actual: Buffer, name: string, config: RunnerConfig, testFile: string, options?: { threshold?: number }): { pass: boolean; message?: string; } { - let expectedPath: string; - const relativeTestFile = path.relative(config.testDir, testFile); - const testAssetsDir = relativeTestFile.replace(/\.spec\.[jt]s/, ''); - if (path.isAbsolute(name)) - expectedPath = name; - else - expectedPath = path.join(config.snapshotDir, testAssetsDir, name); - if (!fs.existsSync(expectedPath)) { - fs.mkdirSync(path.dirname(expectedPath), { recursive: true }); - fs.writeFileSync(expectedPath, actual); - return { - pass: false, - message: expectedPath + ' is missing in golden results, writing actual.' - }; - } - const expected = fs.readFileSync(expectedPath); - const extension = path.extname(expectedPath).substring(1); - const mimeType = extensionToMimeType[extension]; - const comparator = GoldenComparators[mimeType]; - if (!comparator) { - return { - pass: false, - message: 'Failed to find comparator with type ' + mimeType + ': ' + expectedPath, - }; - } - - const result = comparator(actual, expected, mimeType, options); - if (!result) - return { pass: true }; - - if (config.updateSnapshots) { - fs.mkdirSync(path.dirname(expectedPath), { recursive: true }); - fs.writeFileSync(expectedPath, actual); - return { - pass: true, - message: expectedPath + ' running with --update-snapshots, writing actual.' - }; - } - - let actualPath; - let diffPath; - if (path.isAbsolute(name)) { - actualPath = addSuffix(expectedPath, '-actual'); - diffPath = addSuffix(expectedPath, '-diff', result.diffExtension); - } else { - const outputPath = path.join(config.outputDir, testAssetsDir, name); - fs.mkdirSync(path.dirname(outputPath), { recursive: true }); - const expectedPathOut = addSuffix(outputPath, '-expected'); - actualPath = addSuffix(outputPath, '-actual'); - diffPath = addSuffix(outputPath, '-diff', result.diffExtension); - fs.writeFileSync(expectedPathOut, expected); - } - fs.writeFileSync(actualPath, actual); - if (result.diff) - fs.writeFileSync(diffPath, result.diff); - - const output = [ - c.red(`Image comparison failed:`), - ]; - if (result.errorMessage) - output.push(' ' + result.errorMessage); - output.push(''); - output.push(`Expected: ${c.yellow(expectedPath)}`); - output.push(`Received: ${c.yellow(actualPath)}`); - if (result.diff) - output.push(` Diff: ${c.yellow(diffPath)}`); - - return { - pass: false, - message: output.join('\n'), - }; -} - -function addSuffix(filePath: string, suffix: string, customExtension?: string): string { - const dirname = path.dirname(filePath); - const ext = path.extname(filePath); - const name = path.basename(filePath, ext); - return path.join(dirname, name + suffix + (customExtension || ext)); -} diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts deleted file mode 100644 index b318e0942e483..0000000000000 --- a/test-runner/src/index.ts +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright 2019 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as fs from 'fs'; -import * as path from 'path'; -import rimraf from 'rimraf'; -import { promisify } from 'util'; -import './builtin.fixtures'; -import './expect'; -import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT, TestInfo } from './fixtures'; -import { Reporter } from './reporter'; -import { Runner } from './runner'; -import { RunnerConfig } from './runnerConfig'; -import { Suite } from './test'; -import { Matrix, TestCollector } from './testCollector'; -import { installTransform } from './transform'; -import { raceAgainstTimeout } from './util'; -export { parameters, registerParameter } from './fixtures'; -export { Reporter } from './reporter'; -export { RunnerConfig } from './runnerConfig'; -export { Suite, Test } from './test'; - -const removeFolderAsync = promisify(rimraf); - -declare global { - interface WorkerState { - } - - interface TestState { - } - - interface FixtureParameters { - } -} - -const beforeFunctions: Function[] = []; -const afterFunctions: Function[] = []; -let matrix: Matrix = {}; - -global['before'] = (fn: Function) => beforeFunctions.push(fn); -global['after'] = (fn: Function) => afterFunctions.push(fn); -global['matrix'] = (m: Matrix) => matrix = m; - -export function registerFixture(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise, info: TestInfo) => Promise) { - registerFixtureT(name, fn); -} - -export function registerWorkerFixture(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise, config: RunnerConfig) => Promise) { - registerWorkerFixtureT(name, fn); -} - -type RunResult = 'passed' | 'failed' | 'forbid-only' | 'no-tests'; - -export async function run(config: RunnerConfig, files: string[], reporter: Reporter): Promise { - if (!config.trialRun) { - await removeFolderAsync(config.outputDir).catch(e => {}); - fs.mkdirSync(config.outputDir, { recursive: true }); - } - const revertBabelRequire = installTransform(); - let hasSetup = false; - try { - hasSetup = fs.statSync(path.join(config.testDir, 'setup.js')).isFile(); - } catch (e) { - } - try { - hasSetup = hasSetup || fs.statSync(path.join(config.testDir, 'setup.ts')).isFile(); - } catch (e) { - } - if (hasSetup) - require(path.join(config.testDir, 'setup')); - revertBabelRequire(); - - const testCollector = new TestCollector(files, matrix, config); - const suite = testCollector.suite; - if (config.forbidOnly) { - const hasOnly = suite.findTest(t => t._only) || suite.eachSuite(s => s._only); - if (hasOnly) - return 'forbid-only'; - } - - const total = suite.total(); - if (!total) - return 'no-tests'; - const { result, timedOut } = await raceAgainstTimeout(runTests(config, suite, reporter), config.globalTimeout); - if (timedOut) { - reporter.onTimeout(config.globalTimeout); - process.exit(1); - } - return result; -} - -async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) { - // Trial run does not need many workers, use one. - const jobs = (config.trialRun || config.debug) ? 1 : config.jobs; - const runner = new Runner(suite, { ...config, jobs }, reporter); - try { - for (const f of beforeFunctions) - await f(); - await runner.run(); - await runner.stop(); - } finally { - for (const f of afterFunctions) - await f(); - } - return suite.findTest(test => !test._ok()) ? 'failed' : 'passed'; -} diff --git a/test-runner/src/reporter.ts b/test-runner/src/reporter.ts deleted file mode 100644 index 4e5fe34919c83..0000000000000 --- a/test-runner/src/reporter.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { RunnerConfig } from './runnerConfig'; -import { Suite, Test, TestResult } from './test'; - -export interface Reporter { - onBegin(config: RunnerConfig, suite: Suite): void; - onTestBegin(test: Test): void; - onTestStdOut(test: Test, chunk: string | Buffer): void; - onTestStdErr(test: Test, chunk: string | Buffer): void; - onTestEnd(test: Test, result: TestResult): void; - onTimeout(timeout: number): void; - onEnd(): void; -} diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts deleted file mode 100644 index 9f37a496a6bd6..0000000000000 --- a/test-runner/src/reporters/base.ts +++ /dev/null @@ -1,201 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { codeFrameColumns } from '@babel/code-frame'; -import colors from 'colors/safe'; -import fs from 'fs'; -import milliseconds from 'ms'; -import os from 'os'; -import path from 'path'; -import StackUtils from 'stack-utils'; -import terminalLink from 'terminal-link'; -import { Reporter } from '../reporter'; -import { RunnerConfig } from '../runnerConfig'; -import { Suite, Test, TestResult } from '../test'; - -const stackUtils = new StackUtils(); - -export class BaseReporter implements Reporter { - skipped: Test[] = []; - asExpected: Test[] = []; - unexpected = new Set(); - expectedFlaky: Test[] = []; - unexpectedFlaky: Test[] = []; - duration = 0; - startTime: number; - config: RunnerConfig; - suite: Suite; - timeout: number; - - constructor() { - process.on('SIGINT', async () => { - this.epilogue(); - process.exit(130); - }); - } - - onBegin(config: RunnerConfig, suite: Suite) { - this.startTime = Date.now(); - this.config = config; - this.suite = suite; - } - - onTestBegin(test: Test) { - } - - onTestStdOut(test: Test, chunk: string | Buffer) { - if (!this.config.quiet) - process.stdout.write(chunk); - } - - onTestStdErr(test: Test, chunk: string | Buffer) { - if (!this.config.quiet) - process.stderr.write(chunk); - } - - onTestEnd(test: Test, result: TestResult) { - if (result.status === 'skipped') { - this.skipped.push(test); - return; - } - - if (result.status === result.expectedStatus) { - if (test.results.length === 1) { - // as expected from the first attempt - this.asExpected.push(test); - } else { - // as expected after unexpected -> flaky. - if (test.isFlaky()) - this.expectedFlaky.push(test); - else - this.unexpectedFlaky.push(test); - } - return; - } - if (result.status === 'passed' || result.status === 'timedOut' || test.results.length === this.config.retries + 1) { - // We made as many retries as we could, still failing. - this.unexpected.add(test); - } - } - - onTimeout(timeout: number) { - this.timeout = timeout; - } - - onEnd() { - this.duration = Date.now() - this.startTime; - } - - epilogue() { - console.log(''); - - console.log(colors.green(` ${this.asExpected.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); - - if (this.skipped.length) - console.log(colors.yellow(` ${this.skipped.length} skipped`)); - - const filteredUnexpected = [...this.unexpected].filter(t => !t._hasResultWithStatus('timedOut')); - if (filteredUnexpected.length) { - console.log(colors.red(` ${filteredUnexpected.length} failed`)); - console.log(''); - this._printFailures(filteredUnexpected); - } - - const allFlaky = this.expectedFlaky.length + this.unexpectedFlaky.length; - if (allFlaky) { - console.log(colors.red(` ${allFlaky} flaky`)); - if (this.unexpectedFlaky.length) { - console.log(''); - this._printFailures(this.unexpectedFlaky); - } - } - - const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut')); - if (timedOut.length) { - console.log(colors.red(` ${timedOut.length} timed out`)); - console.log(''); - this._printFailures(timedOut); - } - console.log(''); - if (this.timeout) { - console.log(colors.red(` Timed out waiting ${this.timeout / 1000}s for the entire test run`)); - console.log(''); - } - } - - private _printFailures(failures: Test[]) { - failures.forEach((test, index) => { - console.log(this.formatFailure(test, index + 1)); - }); - } - - formatFailure(test: Test, index?: number): string { - const tokens: string[] = []; - const relativePath = path.relative(process.cwd(), test.file); - const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : ''; - const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}${passedUnexpectedlySuffix}`; - tokens.push(colors.bold(colors.red(header))); - for (const result of test.results) { - if (result.status === 'passed') - continue; - if (result.status === 'timedOut') { - tokens.push(''); - tokens.push(indent(colors.red(`Timeout of ${test._timeout}ms exceeded.`), ' ')); - } else { - const stack = result.error.stack; - if (stack) { - tokens.push(''); - const messageLocation = result.error.stack.indexOf(result.error.message); - const preamble = result.error.stack.substring(0, messageLocation + result.error.message.length); - tokens.push(indent(preamble, ' ')); - const position = positionInFile(stack, test.file); - if (position) { - const source = fs.readFileSync(test.file, 'utf8'); - tokens.push(''); - tokens.push(indent(codeFrameColumns(source, { - start: position, - }, - { highlightCode: true} - ), ' ')); - } - tokens.push(''); - tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), ' ')); - } else { - tokens.push(''); - tokens.push(indent(String(result.error), ' ')); - } - } - break; - } - tokens.push(''); - return tokens.join('\n'); - } -} - -function indent(lines: string, tab: string) { - return lines.replace(/^/gm, tab); -} - -function positionInFile(stack: string, file: string): { column: number; line: number; } { - for (const line of stack.split('\n')) { - const parsed = stackUtils.parseLine(line); - if (!parsed) - continue; - if (path.resolve(process.cwd(), parsed.file) === file) - return {column: parsed.column, line: parsed.line}; - } - return null; -} diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts deleted file mode 100644 index 36eb25d08abf4..0000000000000 --- a/test-runner/src/reporters/dot.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import colors from 'colors/safe'; -import { BaseReporter } from './base'; -import { Test, TestResult } from '../test'; - -class DotReporter extends BaseReporter { - onTestEnd(test: Test, result: TestResult) { - super.onTestEnd(test, result); - switch (result.status) { - case 'skipped': process.stdout.write(colors.yellow('∘')); break; - case 'passed': process.stdout.write(result.status === result.expectedStatus ? colors.green('·') : colors.red('P')); break; - case 'failed': process.stdout.write(result.status === result.expectedStatus ? colors.green('f') : colors.red('F')); break; - case 'timedOut': process.stdout.write(colors.red('T')); break; - } - } - - onTimeout(timeout) { - super.onTimeout(timeout); - this.onEnd(); - } - - onEnd() { - super.onEnd(); - process.stdout.write('\n'); - this.epilogue(); - } -} - -export default DotReporter; diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts deleted file mode 100644 index e6fcc7e809dee..0000000000000 --- a/test-runner/src/reporters/json.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { BaseReporter } from './base'; -import { Suite, Test, TestResult } from '../test'; -import * as fs from 'fs'; - -class JSONReporter extends BaseReporter { - onTimeout(timeout) { - super.onTimeout(timeout); - this.onEnd(); - } - - onEnd() { - super.onEnd(); - const result = { - config: this.config, - suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) - }; - const report = JSON.stringify(result, undefined, 2); - if (process.env.PWRUNNER_JSON_REPORT) - fs.writeFileSync(process.env.PWRUNNER_JSON_REPORT, report); - else - console.log(report); - } - - private _serializeSuite(suite: Suite): any { - if (!suite.findTest(test => true)) - return null; - const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s); - return { - title: suite.title, - file: suite.file, - configuration: suite.configuration, - tests: suite.tests.map(test => this._serializeTest(test)), - suites: suites.length ? suites : undefined - }; - } - - private _serializeTest(test: Test): any { - return { - title: test.title, - file: test.file, - only: test.isOnly(), - slow: test.isSlow(), - timeout: test.timeout(), - results: test.results.map(r => this._serializeTestResult(r)) - }; - } - - private _serializeTestResult(result: TestResult): any { - return { - status: result.status, - duration: result.duration, - error: result.error, - stdout: result.stdout.map(s => stdioEntry(s)), - stderr: result.stderr.map(s => stdioEntry(s)), - data: result.data - }; - } -} - -function stdioEntry(s: string | Buffer): any { - if (typeof s === 'string') - return { text: s }; - return { buffer: s.toString('base64') }; -} - -export default JSONReporter; diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts deleted file mode 100644 index 4d485cd1847d1..0000000000000 --- a/test-runner/src/reporters/list.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import colors from 'colors/safe'; -import { BaseReporter } from './base'; -import { RunnerConfig } from '../runnerConfig'; -import { Suite, Test, TestResult } from '../test'; - -class ListReporter extends BaseReporter { - _failure = 0; - - onBegin(config: RunnerConfig, suite: Suite) { - super.onBegin(config, suite); - console.log(); - } - - onTestBegin(test: Test) { - super.onTestBegin(test); - process.stdout.write(' ' + colors.gray(test.fullTitle() + ': ')); - } - - onTestEnd(test: Test, result: TestResult) { - super.onTestEnd(test, result); - let text = ''; - if (result.status === 'skipped') { - text = colors.green(' - ') + colors.cyan(test.fullTitle()); - } else { - const statusMark = result.status === 'passed' ? colors.green(' ✓ ') : colors.red(' x '); - if (result.status === result.expectedStatus) - text = '\u001b[2K\u001b[0G' + statusMark + colors.gray(test.fullTitle()); - else - text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle()); - } - process.stdout.write(text + '\n'); - } - - onEnd() { - super.onEnd(); - process.stdout.write('\n'); - this.epilogue(); - } -} - -export default ListReporter; diff --git a/test-runner/src/reporters/multiplexer.ts b/test-runner/src/reporters/multiplexer.ts deleted file mode 100644 index 0b939b68adbfc..0000000000000 --- a/test-runner/src/reporters/multiplexer.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { RunnerConfig } from '../runnerConfig'; -import { Suite, Test, TestResult } from '../test'; -import { Reporter } from '../reporter'; - -export class Multiplexer implements Reporter { - private _reporters: Reporter[]; - - constructor(reporters: Reporter[]) { - this._reporters = reporters; - } - - onBegin(config: RunnerConfig, suite: Suite) { - for (const reporter of this._reporters) - reporter.onBegin(config, suite); - } - - onTestBegin(test: Test) { - for (const reporter of this._reporters) - reporter.onTestBegin(test); - } - - onTestStdOut(test: Test, chunk: string | Buffer) { - for (const reporter of this._reporters) - reporter.onTestStdOut(test, chunk); - } - - onTestStdErr(test: Test, chunk: string | Buffer) { - for (const reporter of this._reporters) - reporter.onTestStdErr(test, chunk); - } - - onTestEnd(test: Test, result: TestResult) { - for (const reporter of this._reporters) - reporter.onTestEnd(test, result); - } - - onTimeout(timeout: number) { - for (const reporter of this._reporters) - reporter.onTimeout(timeout); - } - - onEnd() { - for (const reporter of this._reporters) - reporter.onEnd(); - } -} diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts deleted file mode 100644 index 2682e99405bf1..0000000000000 --- a/test-runner/src/reporters/pytest.ts +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import colors from 'colors/safe'; -import milliseconds from 'ms'; -import * as path from 'path'; -import { Test, Suite, Configuration, TestResult } from '../test'; -import { BaseReporter } from './base'; -import { RunnerConfig } from '../runnerConfig'; - -const cursorPrevLine = '\u001B[F'; -const eraseLine = '\u001B[2K'; - -type Row = { - id: string; - relativeFile: string; - configuration: string; - ordinal: number; - track: string[]; - total: number; - failed: boolean; - startTime: number; - finishTime: number; -}; - -const statusRows = 2; - -class PytestReporter extends BaseReporter { - private _rows = new Map(); - private _suiteIds = new Map(); - private _lastOrdinal = 0; - private _visibleRows: number; - private _total: number; - private _progress: string[] = []; - private _throttler = new Throttler(250, () => this._repaint()); - - onBegin(config: RunnerConfig, rootSuite: Suite) { - super.onBegin(config, rootSuite); - this._total = rootSuite.total(); - - const jobs = Math.min(config.jobs, rootSuite.suites.length); - this._visibleRows = jobs + Math.min(jobs, 3); // 3 buffer rows for completed (green) workers. - for (let i = 0; i < this._visibleRows + statusRows; ++i) // 4 rows for status - process.stdout.write('\n'); - - for (const s of rootSuite.suites) { - const relativeFile = path.relative(this.config.testDir, s.file); - const configurationString = serializeConfiguration(s.configuration); - const id = relativeFile + `::[${configurationString}]`; - this._suiteIds.set(s, id); - const row = { - id, - relativeFile, - configuration: configurationString, - ordinal: this._lastOrdinal++, - track: [], - total: s.total(), - failed: false, - startTime: 0, - finishTime: 0, - }; - this._rows.set(id, row); - } - } - - onTestBegin(test: Test) { - super.onTestBegin(test); - const row = this._rows.get(this._id(test)); - if (!row.startTime) - row.startTime = Date.now(); - } - - onTestStdOut(test: Test, chunk: string | Buffer) { - this._repaint(chunk); - } - - onTestStdErr(test: Test, chunk: string | Buffer) { - this._repaint(chunk); - } - - onTestEnd(test: Test, result: TestResult) { - super.onTestEnd(test, result); - switch (result.status) { - case 'skipped': { - this._append(test, colors.yellow('∘')); - this._progress.push('S'); - this._throttler.schedule(); - break; - } - case 'passed': { - this._append(test, colors.green('✓')); - this._progress.push('P'); - this._throttler.schedule(); - break; - } - case 'failed': - // fall through - case 'timedOut': { - const title = result.status === 'timedOut' ? colors.red('T') : colors.red('F'); - const row = this._append(test, title); - row.failed = true; - this._progress.push('F'); - this._repaint(this.formatFailure(test) + '\n'); - break; - } - } - } - - private _append(test: Test, s: string): Row { - const testId = this._id(test); - const row = this._rows.get(testId); - row.track.push(s); - if (row.track.length === row.total) - row.finishTime = Date.now(); - return row; - } - - private _repaint(prependChunk?: string | Buffer) { - const rowList = [...this._rows.values()]; - const running = rowList.filter(r => r.startTime && !r.finishTime); - const finished = rowList.filter(r => r.finishTime).sort((a, b) => b.finishTime - a.finishTime); - const finishedToPrint = finished.slice(0, this._visibleRows - running.length); - const lines = []; - for (const row of finishedToPrint.concat(running)) { - const remaining = row.total - row.track.length; - const remainder = '·'.repeat(remaining); - let title = row.relativeFile; - if (row.finishTime) { - if (row.failed) - title = colors.red(row.relativeFile); - else - title = colors.green(row.relativeFile); - } - const configuration = ` [${colors.gray(row.configuration)}]`; - lines.push(' ' + title + configuration + ' ' + row.track.join('') + colors.gray(remainder)); - } - - const status = []; - if (this.asExpected.length) - status.push(colors.green(`${this.asExpected.length} as expected`)); - if (this.skipped.length) - status.push(colors.yellow(`${this.skipped.length} skipped`)); - const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut')); - if (this.unexpected.size - timedOut.length) - status.push(colors.red(`${this.unexpected.size - timedOut.length} unexpected failures`)); - if (timedOut.length) - status.push(colors.red(`${timedOut.length} timed out`)); - status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`)); - - for (let i = lines.length; i < this._visibleRows; ++i) - lines.push(''); - lines.push(this._paintProgress(this._progress.length, this._total)); - lines.push(status.join(' ')); - lines.push(''); - - process.stdout.write((cursorPrevLine + eraseLine).repeat(this._visibleRows + statusRows)); - if (prependChunk) - process.stdout.write(prependChunk); - process.stdout.write(lines.join('\n')); - } - - private _id(test: Test): string { - for (let suite = test.parent; suite; suite = suite.parent) { - if (this._suiteIds.has(suite)) - return this._suiteIds.get(suite); - } - return ''; - } - - private _paintProgress(worked: number, total: number) { - const length = Math.min(total, 80); - const cellSize = Math.ceil(total / length); - const cellNum = (total / cellSize) | 0; - const bars: string[] = []; - for (let i = 0; i < cellNum; ++i) { - let bar = blankBar; - if (worked < cellSize * i) { - bars.push(bar); - continue; - } - bar = greenBar; - for (let j = i * cellSize; j < worked && j < (i + 1) * cellSize; ++j) { - if (worked < j) - continue; - if (this._progress[j] === 'F') { - bar = redBar; - break; - } - if (this._progress[j] === 'S') { - bar = yellowBar; - break; - } - } - bars.push(bar); - } - return '[' + bars.join('') + '] ' + worked + '/' + total; - } -} - -const blankBar = '-'; -const redBar = colors.red('▇'); -const greenBar = colors.green('▇'); -const yellowBar = colors.yellow('▇'); - -function serializeConfiguration(configuration: Configuration): string { - const tokens = []; - for (const { name, value } of configuration) - tokens.push(`${name}=${value}`); - return tokens.join(', '); -} - -class Throttler { - private _timeout: number; - private _callback: () => void; - private _lastFire = 0; - private _timer: NodeJS.Timeout | null = null; - - constructor(timeout: number, callback: () => void) { - this._timeout = timeout; - this._callback = callback; - } - - schedule() { - const time = Date.now(); - const timeRemaining = this._lastFire + this._timeout - time; - if (timeRemaining <= 0) { - this._fire(); - return; - } - if (!this._timer) - this._timer = setTimeout(() => this._fire(), timeRemaining); - } - - private _fire() { - this._timer = null; - this._lastFire = Date.now(); - this._callback(); - } -} - -export default PytestReporter; diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts deleted file mode 100644 index decf4f6290951..0000000000000 --- a/test-runner/src/runner.ts +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import child_process from 'child_process'; -import crypto from 'crypto'; -import path from 'path'; -import { EventEmitter } from 'events'; -import { lookupRegistrations, FixturePool } from './fixtures'; -import { Suite, Test, TestResult } from './test'; -import { TestRunnerEntry } from './testRunner'; -import { RunnerConfig } from './runnerConfig'; -import { Reporter } from './reporter'; - -export class Runner { - private _workers = new Set(); - private _freeWorkers: Worker[] = []; - private _workerClaimers: (() => void)[] = []; - - private _testById = new Map(); - private _queue: TestRunnerEntry[] = []; - private _stopCallback: () => void; - readonly _config: RunnerConfig; - private _suite: Suite; - private _reporter: Reporter; - - constructor(suite: Suite, config: RunnerConfig, reporter: Reporter) { - this._config = config; - this._reporter = reporter; - - this._suite = suite; - for (const suite of this._suite.suites) { - suite.findTest(test => { - this._testById.set(test._id, { test, result: test._appendResult() }); - }); - } - - if (process.stdout.isTTY) { - const total = suite.total(); - console.log(); - const jobs = Math.min(config.jobs, suite.suites.length); - console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}`); - } - } - - _filesSortedByWorkerHash(): TestRunnerEntry[] { - const result: TestRunnerEntry[] = []; - for (const suite of this._suite.suites) { - const ids: string[] = []; - suite.findTest(test => ids.push(test._id) && false); - if (!ids.length) - continue; - result.push({ - ids, - file: suite.file, - configuration: suite.configuration, - configurationString: suite._configurationString, - hash: suite._configurationString + '@' + computeWorkerHash(suite.file) - }); - } - result.sort((a, b) => a.hash < b.hash ? -1 : (a.hash === b.hash ? 0 : 1)); - return result; - } - - async run() { - this._reporter.onBegin(this._config, this._suite); - this._queue = this._filesSortedByWorkerHash(); - // Loop in case job schedules more jobs - while (this._queue.length) - await this._dispatchQueue(); - this._reporter.onEnd(); - } - - async _dispatchQueue() { - const jobs = []; - while (this._queue.length) { - const entry = this._queue.shift(); - const requiredHash = entry.hash; - let worker = await this._obtainWorker(); - while (worker.hash && worker.hash !== requiredHash) { - this._restartWorker(worker); - worker = await this._obtainWorker(); - } - jobs.push(this._runJob(worker, entry)); - } - await Promise.all(jobs); - } - - async _runJob(worker: Worker, entry: TestRunnerEntry) { - worker.run(entry); - let doneCallback; - const result = new Promise(f => doneCallback = f); - worker.once('done', params => { - // We won't file remaining if: - // - there are no remaining - // - we are here not because something failed - // - no unrecoverable worker error - if (!params.remaining.length && !params.failedTestId && !params.fatalError) { - this._workerAvailable(worker); - doneCallback(); - return; - } - - // When worker encounters error, we will restart it. - this._restartWorker(worker); - - // In case of fatal error, we are done with the entry. - if (params.fatalError) { - // Report all the tests are failing with this error. - for (const id of entry.ids) { - const { test, result } = this._testById.get(id); - this._reporter.onTestBegin(test); - result.status = 'failed'; - result.error = params.fatalError; - this._reporter.onTestEnd(test, result); - } - doneCallback(); - return; - } - - const remaining = params.remaining; - - // Only retry expected failures, not passes and only if the test failed. - if (this._config.retries && params.failedTestId) { - const pair = this._testById.get(params.failedTestId); - if (pair.result.expectedStatus === 'passed' && pair.test.results.length < this._config.retries + 1) { - pair.result = pair.test._appendResult(); - remaining.unshift(pair.test._id); - } - } - - if (remaining.length) - this._queue.unshift({ ...entry, ids: remaining }); - - // This job is over, we just scheduled another one. - doneCallback(); - }); - return result; - } - - async _obtainWorker() { - // If there is worker, use it. - if (this._freeWorkers.length) - return this._freeWorkers.pop(); - // If we can create worker, create it. - if (this._workers.size < this._config.jobs) - this._createWorker(); - // Wait for the next available worker. - await new Promise(f => this._workerClaimers.push(f)); - return this._freeWorkers.pop(); - } - - async _workerAvailable(worker) { - this._freeWorkers.push(worker); - if (this._workerClaimers.length) { - const callback = this._workerClaimers.shift(); - callback(); - } - } - - _createWorker() { - const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this); - worker.on('testBegin', params => { - const { test } = this._testById.get(params.id); - test._skipped = params.skipped; - test._flaky = params.flaky; - test._slow = params.slow; - test._expectedStatus = params.expectedStatus; - this._reporter.onTestBegin(test); - }); - worker.on('testEnd', params => { - const workerResult: TestResult = params.result; - // We were accumulating these below. - delete workerResult.stdout; - delete workerResult.stderr; - const { test, result } = this._testById.get(params.id); - Object.assign(result, workerResult); - this._reporter.onTestEnd(test, result); - }); - worker.on('testStdOut', params => { - const chunk = chunkFromParams(params); - const { test, result } = this._testById.get(params.id); - result.stdout.push(chunk); - this._reporter.onTestStdOut(test, chunk); - }); - worker.on('testStdErr', params => { - const chunk = chunkFromParams(params); - const { test, result } = this._testById.get(params.id); - result.stderr.push(chunk); - this._reporter.onTestStdErr(test, chunk); - }); - worker.on('exit', () => { - this._workers.delete(worker); - if (this._stopCallback && !this._workers.size) - this._stopCallback(); - }); - this._workers.add(worker); - worker.init().then(() => this._workerAvailable(worker)); - } - - async _restartWorker(worker) { - await worker.stop(); - this._createWorker(); - } - - async stop() { - const result = new Promise(f => this._stopCallback = f); - for (const worker of this._workers) - worker.stop(); - await result; - } -} - -let lastWorkerId = 0; - -class Worker extends EventEmitter { - runner: Runner; - hash: string; - - constructor(runner) { - super(); - this.runner = runner; - } - - run(entry: TestRunnerEntry) { - } - - stop() { - } -} - -class OopWorker extends Worker { - process: child_process.ChildProcess; - stdout: any[]; - stderr: any[]; - constructor(runner: Runner) { - super(runner); - - this.process = child_process.fork(path.join(__dirname, 'worker.js'), { - detached: false, - env: { - FORCE_COLOR: process.stdout.isTTY ? '1' : '0', - DEBUG_COLORS: process.stdout.isTTY ? '1' : '0', - ...process.env - }, - // Can't pipe since piping slows down termination for some reason. - stdio: ['ignore', 'ignore', 'ignore', 'ipc'] - }); - this.process.on('exit', () => this.emit('exit')); - this.process.on('error', e => {}); // do not yell at a send to dead process. - this.process.on('message', message => { - const { method, params } = message; - this.emit(method, params); - }); - } - - async init() { - this.process.send({ method: 'init', params: { workerId: lastWorkerId++, ...this.runner._config } }); - await new Promise(f => this.process.once('message', f)); // Ready ack - } - - run(entry: TestRunnerEntry) { - this.hash = entry.hash; - this.process.send({ method: 'run', params: { entry, config: this.runner._config } }); - } - - stop() { - this.process.send({ method: 'stop' }); - } -} - -class InProcessWorker extends Worker { - fixturePool: FixturePool; - - constructor(runner: Runner) { - super(runner); - this.fixturePool = require('./testRunner').fixturePool as FixturePool; - } - - async init() { - const { initializeImageMatcher } = require('./expect'); - initializeImageMatcher(this.runner._config); - } - - async run(entry: TestRunnerEntry) { - delete require.cache[entry.file]; - const { TestRunner } = require('./testRunner'); - const testRunner = new TestRunner(entry, this.runner._config, 0); - for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done']) - testRunner.on(event, this.emit.bind(this, event)); - testRunner.run(); - } - - async stop() { - await this.fixturePool.teardownScope('worker'); - this.emit('exit'); - } -} - -function chunkFromParams(params: { testId: string, buffer?: string, text?: string }): string | Buffer { - if (typeof params.text === 'string') - return params.text; - return Buffer.from(params.buffer, 'base64'); -} - -function computeWorkerHash(file: string) { - // At this point, registrationsByFile contains all the files with worker fixture registrations. - // For every test, build the require closure and map each file to fixtures declared in it. - // This collection of fixtures is the fingerprint of the worker setup, a "worker hash". - // Tests with the matching "worker hash" will reuse the same worker. - const hash = crypto.createHash('sha1'); - for (const registration of lookupRegistrations(file, 'worker').values()) - hash.update(registration.location); - return hash.digest('hex'); -} diff --git a/test-runner/src/runnerConfig.ts b/test-runner/src/runnerConfig.ts deleted file mode 100644 index fab686f2f7cf5..0000000000000 --- a/test-runner/src/runnerConfig.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type RunnerConfig = { - debug?: boolean; - forbidOnly?: boolean; - globalTimeout: number; - grep?: string; - jobs: number; - outputDir: string; - quiet?: boolean; - repeatEach: number; - retries: number, - snapshotDir: string; - testDir: string; - timeout: number; - trialRun?: boolean; - updateSnapshots?: boolean; -}; diff --git a/test-runner/src/spec.ts b/test-runner/src/spec.ts deleted file mode 100644 index 978f56a487b0f..0000000000000 --- a/test-runner/src/spec.ts +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Test, Suite } from './test'; -import { installTransform } from './transform'; - -Error.stackTraceLimit = 15; - -function specBuilder(modifiers, specCallback) { - function builder(specs, last) { - const callable = (...args) => { - if (!last || (typeof args[0] === 'string' && typeof args[1] === 'function')) { - // Looks like a body (either it or describe). Assume that last modifier is true. - const newSpecs = { ...specs }; - if (last) - newSpecs[last] = [true]; - return specCallback(newSpecs, ...args); - } - const newSpecs = { ...specs }; - newSpecs[last] = args; - return builder(newSpecs, null); - }; - return new Proxy(callable, { - get: (obj, prop) => { - if (typeof prop === 'string' && modifiers.includes(prop)) { - const newSpecs = { ...specs }; - // Modifier was not called, assume true. - if (last) - newSpecs[last] = [true]; - return builder(newSpecs, prop); - } - return obj[prop]; - }, - }); - } - return builder({}, null); -} - -export function spec(suite: Suite, file: string, timeout: number): () => void { - const suites = [suite]; - suite.file = file; - - const it = specBuilder(['_skip', '_only'], (specs: any, title: string, metaFn: (test: Test) => void | Function, fn?: Function) => { - const suite = suites[0]; - if (typeof fn !== 'function') { - fn = metaFn; - metaFn = null; - } - const test = new Test(title, fn); - if (metaFn) - metaFn(test); - test.file = file; - test._timeout = timeout; - const only = specs._only && specs._only[0]; - if (only) - test._only = true; - if (!only && specs._skip && specs._skip[0]) - test._skipped = true; - suite._addTest(test); - return test; - }); - - const describe = specBuilder(['_skip', '_only'], (specs: any, title: string, metaFn: (suite: Suite) => void | Function, fn?: Function) => { - if (typeof fn !== 'function') { - fn = metaFn; - metaFn = null; - } - const child = new Suite(title, suites[0]); - if (metaFn) - metaFn(child); - suites[0]._addSuite(child); - child.file = file; - const only = specs._only && specs._only[0]; - if (only) - child._only = true; - if (!only && specs._skip && specs._skip[0]) - child._skipped = true; - suites.unshift(child); - fn(); - suites.shift(); - }); - - (global as any).beforeEach = fn => suite._addHook('beforeEach', fn); - (global as any).afterEach = fn => suite._addHook('afterEach', fn); - (global as any).beforeAll = fn => suite._addHook('beforeAll', fn); - (global as any).afterAll = fn => suite._addHook('afterAll', fn); - (global as any).describe = describe; - (global as any).fdescribe = describe._only(true); - (global as any).xdescribe = describe._skip(true); - (global as any).it = it; - (global as any).fit = it._only(true); - (global as any).xit = it._skip(true); - - return installTransform(); -} diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts deleted file mode 100644 index 9e58eb0dc2d86..0000000000000 --- a/test-runner/src/test.ts +++ /dev/null @@ -1,308 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export type Configuration = { name: string, value: string }[]; - -type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; - -export class Runnable { - title: string; - file: string; - parent?: Suite; - - _only = false; - _skipped = false; - _flaky = false; - _slow = false; - _expectedStatus: TestStatus = 'passed'; - - isOnly(): boolean { - return this._only; - } - - isSlow(): boolean { - return this._slow; - } - - slow(): void; - slow(condition: boolean): void; - slow(description: string): void; - slow(condition: boolean, description: string): void; - slow(arg?: boolean | string, description?: string) { - const { condition } = this._interpretCondition(arg, description); - if (condition) - this._slow = true; - } - - skip(): void; - skip(condition: boolean): void; - skip(description: string): void; - skip(condition: boolean, description: string): void; - skip(arg?: boolean | string, description?: string) { - const { condition } = this._interpretCondition(arg, description); - if (condition) - this._skipped = true; - } - - fixme(): void; - fixme(condition: boolean): void; - fixme(description: string): void; - fixme(condition: boolean, description: string): void; - fixme(arg?: boolean | string, description?: string) { - const { condition } = this._interpretCondition(arg, description); - if (condition) - this._skipped = true; - } - - flaky(): void; - flaky(condition: boolean): void; - flaky(description: string): void; - flaky(condition: boolean, description: string): void; - flaky(arg?: boolean | string, description?: string) { - const { condition } = this._interpretCondition(arg, description); - if (condition) - this._flaky = true; - } - - fail(): void; - fail(condition: boolean): void; - fail(description: string): void; - fail(condition: boolean, description: string): void; - fail(arg?: boolean | string, description?: string) { - const { condition } = this._interpretCondition(arg, description); - if (condition) - this._expectedStatus = 'failed'; - } - - private _interpretCondition(arg?: boolean | string, description?: string): { condition: boolean, description?: string } { - if (arg === undefined && description === undefined) - return { condition: true }; - if (typeof arg === 'string') - return { condition: true, description: arg }; - return { condition: !!arg, description }; - } - - _isSkipped(): boolean { - return this._skipped || (this.parent && this.parent._isSkipped()); - } - - _isSlow(): boolean { - return this._slow || (this.parent && this.parent._isSlow()); - } - - isFlaky(): boolean { - return this._flaky || (this.parent && this.parent.isFlaky()); - } - - titlePath(): string[] { - if (!this.parent) - return []; - return [...this.parent.titlePath(), this.title]; - } - - fullTitle(): string { - return this.titlePath().join(' '); - } - - _copyFrom(other: Runnable) { - this.file = other.file; - this._only = other._only; - this._flaky = other._flaky; - this._skipped = other._skipped; - this._slow = other._slow; - } -} - -export class Test extends Runnable { - fn: Function; - results: TestResult[] = []; - _id: string; - _overriddenFn: Function; - _startTime: number; - _timeout = 0; - - constructor(title: string, fn: Function) { - super(); - this.title = title; - this.fn = fn; - } - - _appendResult(): TestResult { - const result: TestResult = { - duration: 0, - expectedStatus: 'passed', - stdout: [], - stderr: [], - data: {} - }; - this.results.push(result); - return result; - } - - timeout(): number { - return this._timeout; - } - - _ok(): boolean { - if (this._isSkipped()) - return true; - const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus); - if (!hasFailedResults) - return true; - if (!this.isFlaky()) - return false; - const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus); - return hasPassedResults; - } - - _hasResultWithStatus(status: TestStatus): boolean { - return !!this.results.find(r => r.status === status); - } - - _clone(): Test { - const test = new Test(this.title, this.fn); - test._copyFrom(this); - test._timeout = this._timeout; - test._overriddenFn = this._overriddenFn; - return test; - } -} - -export type TestResult = { - duration: number; - status?: TestStatus; - expectedStatus: TestStatus; - error?: any; - stdout: (string | Buffer)[]; - stderr: (string | Buffer)[]; - data: any; -} - -export class Suite extends Runnable { - suites: Suite[] = []; - tests: Test[] = []; - configuration: Configuration; - _configurationString: string; - - _hooks: { type: string, fn: Function } [] = []; - _entries: (Suite | Test)[] = []; - - constructor(title: string, parent?: Suite) { - super(); - this.title = title; - this.parent = parent; - } - - total(): number { - let count = 0; - this.findTest(fn => { - ++count; - }); - return count; - } - - _addTest(test: Test) { - test.parent = this; - this.tests.push(test); - this._entries.push(test); - } - - _addSuite(suite: Suite) { - suite.parent = this; - this.suites.push(suite); - this._entries.push(suite); - } - - eachSuite(fn: (suite: Suite) => boolean | void): boolean { - for (const suite of this.suites) { - if (suite.eachSuite(fn)) - return true; - } - return false; - } - - findTest(fn: (test: Test) => boolean | void): boolean { - for (const suite of this.suites) { - if (suite.findTest(fn)) - return true; - } - for (const test of this.tests) { - if (fn(test)) - return true; - } - return false; - } - - _clone(): Suite { - const suite = new Suite(this.title); - suite._copyFrom(this); - return suite; - } - - _renumber() { - let ordinal = 0; - this.findTest((test: Test) => { - // All tests are identified with their ordinals. - test._id = `${ordinal++}@${this.file}::[${this._configurationString}]`; - }); - } - - _addHook(type: string, fn: any) { - this._hooks.push({ type, fn }); - } - - _hasTestsToRun(): boolean { - let found = false; - this.findTest(test => { - if (!test._isSkipped()) { - found = true; - return true; - } - }); - return found; - } -} - -export function serializeConfiguration(configuration: Configuration): string { - const tokens = []; - for (const { name, value } of configuration) - tokens.push(`${name}=${value}`); - return tokens.join(', '); -} - -export function serializeError(error: Error | any): any { - if (error instanceof Error) { - return { - message: error.message, - stack: error.stack - }; - } - return trimCycles(error); -} - -function trimCycles(obj: any): any { - const cache = new Set(); - return JSON.parse( - JSON.stringify(obj, function(key, value) { - if (typeof value === 'object' && value !== null) { - if (cache.has(value)) - return '' + value; - cache.add(value); - } - return value; - }) - ); -} diff --git a/test-runner/src/testCollector.ts b/test-runner/src/testCollector.ts deleted file mode 100644 index a26c5d537b9aa..0000000000000 --- a/test-runner/src/testCollector.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import path from 'path'; -import { fixturesForCallback } from './fixtures'; -import { Test, Suite, serializeConfiguration } from './test'; -import { spec } from './spec'; -import { RunnerConfig } from './runnerConfig'; - - -export type Matrix = { - [key: string]: string[] -}; - -export class TestCollector { - suite: Suite; - - private _matrix: Matrix; - private _config: RunnerConfig; - private _grep: RegExp; - private _hasOnly: boolean; - - constructor(files: string[], matrix: Matrix, config: RunnerConfig) { - this._matrix = matrix; - this._config = config; - this.suite = new Suite(''); - if (config.grep) { - const match = config.grep.match(/^\/(.*)\/(g|i|)$|.*/); - this._grep = new RegExp(match[1] || match[0], match[2]); - } - - for (const file of files) - this._addFile(file); - - this._hasOnly = this._filterOnly(this.suite); - } - - hasOnly() { - return this._hasOnly; - } - - private _addFile(file: string) { - const suite = new Suite(''); - const revertBabelRequire = spec(suite, file, this._config.timeout); - require(file); - revertBabelRequire(); - - const workerGeneratorConfigurations = new Map(); - - suite.findTest((test: Test) => { - // Get all the fixtures that the test needs. - const fixtures = fixturesForCallback(test.fn); - - // For generator fixtures, collect all variants of the fixture values - // to build different workers for them. - const generatorConfigurations = []; - for (const name of fixtures) { - const values = this._matrix[name]; - if (!values) - continue; - const state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]]; - generatorConfigurations.length = 0; - for (const gen of state) { - for (const value of values) - generatorConfigurations.push([...gen, { name, value }]); - } - } - - // No generator fixtures for test, include empty set. - if (!generatorConfigurations.length) - generatorConfigurations.push([]); - - for (const configuration of generatorConfigurations) { - // Serialize configuration as readable string, we will use it as a hash. - const configurationString = serializeConfiguration(configuration); - // Allocate worker for this configuration, add test into it. - if (!workerGeneratorConfigurations.has(configurationString)) - workerGeneratorConfigurations.set(configurationString, { configuration, configurationString, tests: new Set() }); - workerGeneratorConfigurations.get(configurationString).tests.add(test); - } - }); - - // Clone the suite as many times as we have repeat each. - for (let i = 0; i < this._config.repeatEach; ++i) { - // Clone the suite as many times as there are worker hashes. - // Only include the tests that requested these generations. - for (const [hash, {configuration, configurationString, tests}] of workerGeneratorConfigurations.entries()) { - const clone = this._cloneSuite(suite, tests); - this.suite._addSuite(clone); - clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '') + ' ' + (i ? ` #repeat-${i}#` : ''); - clone.configuration = configuration; - clone._configurationString = configurationString + `#repeat-${i}#`; - clone._renumber(); - } - } - } - - private _cloneSuite(suite: Suite, tests: Set) { - const copy = suite._clone(); - for (const entry of suite._entries) { - if (entry instanceof Suite) { - copy._addSuite(this._cloneSuite(entry, tests)); - } else { - const test = entry; - if (!tests.has(test)) - continue; - if (this._grep && !this._grep.test(test.fullTitle())) - continue; - const testCopy = test._clone(); - copy._addTest(testCopy); - } - } - return copy; - } - - private _filterOnly(suite) { - const onlySuites = suite.suites.filter((child: Suite) => this._filterOnly(child) || child._only); - const onlyTests = suite.tests.filter((test: Test) => test._only); - if (onlySuites.length || onlyTests.length) { - suite.suites = onlySuites; - suite.tests = onlyTests; - return true; - } - return false; - } -} diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts deleted file mode 100644 index c8247d203831e..0000000000000 --- a/test-runner/src/testRunner.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { FixturePool, rerunRegistrations, setParameters, TestInfo } from './fixtures'; -import { EventEmitter } from 'events'; -import { setCurrentTestFile } from './expect'; -import { Test, Suite, Configuration, serializeError, TestResult } from './test'; -import { spec } from './spec'; -import { RunnerConfig } from './runnerConfig'; -import * as util from 'util'; - -export const fixturePool = new FixturePool(); - -export type TestRunnerEntry = { - file: string; - ids: string[]; - configurationString: string; - configuration: Configuration; - hash: string; -}; - -function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: string } { - if (chunk instanceof Buffer) - return { buffer: chunk.toString('base64') }; - if (typeof chunk !== 'string') - return { text: util.inspect(chunk) }; - return { text: chunk }; -} - -export class TestRunner extends EventEmitter { - private _failedTestId: string | undefined; - private _fatalError: any | undefined; - private _ids: Set; - private _remaining: Set; - private _trialRun: any; - private _parsedGeneratorConfiguration: any = {}; - private _config: RunnerConfig; - private _timeout: number; - private _testId: string | null; - private _stdOutBuffer: (string | Buffer)[] = []; - private _stdErrBuffer: (string | Buffer)[] = []; - private _testResult: TestResult | null = null; - private _suite: Suite; - private _loaded = false; - - constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) { - super(); - this._suite = new Suite(''); - this._suite.file = entry.file; - this._suite._configurationString = entry.configurationString; - this._ids = new Set(entry.ids); - this._remaining = new Set(entry.ids); - this._trialRun = config.trialRun; - this._timeout = config.timeout; - this._config = config; - for (const {name, value} of entry.configuration) - this._parsedGeneratorConfiguration[name] = value; - this._parsedGeneratorConfiguration['parallelIndex'] = workerId; - setCurrentTestFile(this._suite.file); - } - - stop() { - this._trialRun = true; - } - - unhandledError(error: Error | any) { - if (this._testResult) { - this._testResult.status = 'failed'; - this._testResult.error = serializeError(error); - this._failedTestId = this._testId; - this.emit('testEnd', { - id: this._testId, - result: this._testResult - }); - this._testResult = null; - } else if (!this._loaded) { - // No current test - fatal error. - this._fatalError = serializeError(error); - } - this._reportDone(); - } - - stdout(chunk: string | Buffer) { - this._stdOutBuffer.push(chunk); - if (!this._testId) - return; - for (const c of this._stdOutBuffer) - this.emit('testStdOut', { id: this._testId, ...chunkToParams(c) }); - this._stdOutBuffer = []; - } - - stderr(chunk: string | Buffer) { - this._stdErrBuffer.push(chunk); - if (!this._testId) - return; - for (const c of this._stdErrBuffer) - this.emit('testStdErr', { id: this._testId, ...chunkToParams(c) }); - this._stdErrBuffer = []; - } - - async run() { - setParameters(this._parsedGeneratorConfiguration); - - const revertBabelRequire = spec(this._suite, this._suite.file, this._timeout); - require(this._suite.file); - revertBabelRequire(); - this._suite._renumber(); - this._loaded = true; - - rerunRegistrations(this._suite.file, 'test'); - await this._runSuite(this._suite); - this._reportDone(); - } - - private async _runSuite(suite: Suite) { - try { - await this._runHooks(suite, 'beforeAll', 'before'); - } catch (e) { - this._fatalError = serializeError(e); - this._reportDone(); - } - for (const entry of suite._entries) { - if (entry instanceof Suite) - await this._runSuite(entry); - else - await this._runTest(entry); - } - try { - await this._runHooks(suite, 'afterAll', 'after'); - } catch (e) { - this._fatalError = serializeError(e); - this._reportDone(); - } - } - - private async _runTest(test: Test) { - if (this._failedTestId) - return false; - if (this._ids.size && !this._ids.has(test._id)) - return; - this._remaining.delete(test._id); - - const id = test._id; - this._testId = id; - // We only know resolved skipped/flaky value in the worker, - // send it to the runner. - test._skipped = test._isSkipped(); - test._flaky = test.isFlaky(); - test._slow = test._isSlow(); - this.emit('testBegin', { - id, - skipped: test._skipped, - flaky: test._flaky, - slow: test._slow - }); - - const result: TestResult = { - duration: 0, - status: 'passed', - expectedStatus: test._expectedStatus, - stdout: [], - stderr: [], - data: {} - }; - this._testResult = result; - - if (test._skipped) { - result.status = 'skipped'; - this.emit('testEnd', { id, result }); - return; - } - - const startTime = Date.now(); - try { - const testInfo = { config: this._config, test, result }; - if (!this._trialRun) { - await this._runHooks(test.parent, 'beforeEach', 'before', testInfo); - const timeout = test._isSlow() ? this._timeout * 3 : this._timeout; - await fixturePool.runTestWithFixturesAndTimeout(test.fn, timeout, testInfo); - await this._runHooks(test.parent, 'afterEach', 'after', testInfo); - } else { - result.status = result.expectedStatus; - } - } catch (error) { - // Error in the test fixture teardown. - result.status = 'failed'; - result.error = serializeError(error); - } - result.duration = Date.now() - startTime; - if (this._testResult) { - // We could have reported end due to an unhandled exception. - this.emit('testEnd', { id, result }); - } - if (result.status !== 'passed') - this._failedTestId = this._testId; - this._testResult = null; - this._testId = null; - } - - private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after', testInfo?: TestInfo) { - if (!suite._hasTestsToRun()) - return; - const all = []; - for (let s = suite; s; s = s.parent) { - const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn); - all.push(...funcs.reverse()); - } - if (dir === 'before') - all.reverse(); - for (const hook of all) - await fixturePool.resolveParametersAndRun(hook, this._config, testInfo); - } - - private _reportDone() { - this.emit('done', { - failedTestId: this._failedTestId, - fatalError: this._fatalError, - remaining: [...this._remaining], - }); - } -} diff --git a/test-runner/src/transform.ts b/test-runner/src/transform.ts deleted file mode 100644 index b97e16fa7811d..0000000000000 --- a/test-runner/src/transform.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as crypto from 'crypto'; -import * as os from 'os'; -import * as path from 'path'; -import * as fs from 'fs'; -import * as pirates from 'pirates'; -import * as babel from '@babel/core'; -import * as sourceMapSupport from 'source-map-support'; - -const version = 2; -const cacheDir = path.join(os.tmpdir(), 'playwright-transform-cache'); -const sourceMaps: Map = new Map(); - -sourceMapSupport.install({ - environment: 'node', - handleUncaughtExceptions: false, - retrieveSourceMap(source) { - if (!sourceMaps.has(source)) - return null; - const sourceMapPath = sourceMaps.get(source); - if (!fs.existsSync(sourceMapPath)) - return null; - return { - map: JSON.parse(fs.readFileSync(sourceMapPath, 'utf-8')), - url: source - }; - } -}); - -function calculateCachePath(content: string, filePath: string): string { - const hash = crypto.createHash('sha1').update(content).update(filePath).update(String(version)).digest('hex'); - const fileName = path.basename(filePath, path.extname(filePath)).replace(/\W/g, '') + '_' + hash; - return path.join(cacheDir, hash[0] + hash[1], fileName); -} - -export function installTransform(): () => void { - return pirates.addHook((code, filename) => { - const cachePath = calculateCachePath(code, filename); - const codePath = cachePath + '.js'; - const sourceMapPath = cachePath + '.map'; - sourceMaps.set(filename, sourceMapPath); - if (fs.existsSync(codePath)) - return fs.readFileSync(codePath, 'utf8'); - - const result = babel.transformFileSync(filename, { - presets: [ - ['@babel/preset-env', { targets: {node: 'current'} }], - ['@babel/preset-typescript', { onlyRemoveTypeImports: true }], - ], - sourceMaps: true, - }); - if (result.code) { - fs.mkdirSync(path.dirname(cachePath), {recursive: true}); - if (result.map) - fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8'); - fs.writeFileSync(codePath, result.code, 'utf8'); - } - return result.code; - }, { - exts: ['.ts'] - }); -} diff --git a/test-runner/src/util.ts b/test-runner/src/util.ts deleted file mode 100644 index 0e9c862088489..0000000000000 --- a/test-runner/src/util.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export async function raceAgainstTimeout(promise: Promise, timeout: number): Promise<{ result?: T, timedOut?: boolean }> { - if (!timeout) - return { result: await promise }; - - let timer: NodeJS.Timer; - let done = false; - let fulfill: (t: { result?: T, timedOut?: boolean }) => void; - let reject: (e: Error) => void; - const result = new Promise((f, r) => { - fulfill = f; - reject = r; - }); - setTimeout(() => { - done = true; - fulfill({ timedOut: true }); - }, timeout); - promise.then(result => { - clearTimeout(timer); - if (!done) { - done = true; - fulfill({ result }); - } - }).catch(e => { - clearTimeout(timer); - if (!done) - reject(e); - }); - return result; -} diff --git a/test-runner/src/worker.ts b/test-runner/src/worker.ts deleted file mode 100644 index 1d72ce72d9676..0000000000000 --- a/test-runner/src/worker.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { initializeImageMatcher } from './expect'; -import { TestRunner, fixturePool } from './testRunner'; -import { Console } from 'console'; - -let closed = false; - -sendMessageToParent('ready'); - -global.console = new Console({ - stdout: process.stdout, - stderr: process.stderr, - colorMode: process.env.FORCE_COLOR === '1', -}); - -process.stdout.write = chunk => { - if (testRunner) - testRunner.stdout(chunk); - return true; -}; - -process.stderr.write = chunk => { - if (testRunner) - testRunner.stderr(chunk); - return true; -}; - -process.on('disconnect', gracefullyCloseAndExit); -process.on('SIGINT',() => {}); -process.on('SIGTERM',() => {}); - -let workerId: number; -let testRunner: TestRunner; - -process.on('unhandledRejection', (reason, promise) => { - if (testRunner) - testRunner.unhandledError(reason); -}); - -process.on('uncaughtException', error => { - if (testRunner) - testRunner.unhandledError(error); -}); - -process.on('message', async message => { - if (message.method === 'init') { - workerId = message.params.workerId; - initializeImageMatcher(message.params); - return; - } - if (message.method === 'stop') { - await gracefullyCloseAndExit(); - return; - } - if (message.method === 'run') { - testRunner = new TestRunner(message.params.entry, message.params.config, workerId); - for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done']) - testRunner.on(event, sendMessageToParent.bind(null, event)); - await testRunner.run(); - testRunner = null; - } -}); - -async function gracefullyCloseAndExit() { - if (closed) - return; - closed = true; - // Force exit after 30 seconds. - setTimeout(() => process.exit(0), 30000); - // Meanwhile, try to gracefully close all browsers. - if (testRunner) - testRunner.stop(); - await fixturePool.teardownScope('worker'); - process.exit(0); -} - -function sendMessageToParent(method, params = {}) { - if (closed) - return; - try { - process.send({ method, params }); - } catch (e) { - // Can throw when closing. - } -} diff --git a/test-runner/test/assets/allow-flaky.js b/test-runner/test/assets/allow-flaky.js deleted file mode 100644 index 78677d718ceaa..0000000000000 --- a/test-runner/test/assets/allow-flaky.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const fs = require('fs'); -const path = require('path'); - -it('flake', test => { - test.flaky(); -}, async ({}) => { - try { - fs.readFileSync(path.join(__dirname, '..', 'test-results', 'allow-flaky.txt')); - } catch (e) { - // First time this fails. - fs.writeFileSync(path.join(__dirname, '..', 'test-results', 'allow-flaky.txt'), 'TRUE'); - expect(true).toBe(false); - } -}); diff --git a/test-runner/test/assets/expected-failure.js b/test-runner/test/assets/expected-failure.js deleted file mode 100644 index b265756e75b17..0000000000000 --- a/test-runner/test/assets/expected-failure.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../lib'); - -it('fails', test => test.fail(), () => { - expect(1 + 1).toBe(3); -}); - -it('non-empty remaining',() => { - expect(1 + 1).toBe(2); -}); diff --git a/test-runner/test/assets/fixture-timeout.js b/test-runner/test/assets/fixture-timeout.js deleted file mode 100644 index 8836bd0dd9d42..0000000000000 --- a/test-runner/test/assets/fixture-timeout.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { registerFixture } = require('../../'); - -registerFixture('timeout', async ({}, runTest) => { - await runTest(); - await new Promise(f => setTimeout(f, 100000)); -}); - -it('fixture timeout', async({timeout}) => { - expect(1).toBe(1); -}); - -it('failing fixture timeout', async({timeout}) => { - expect(1).toBe(2); -}); diff --git a/test-runner/test/assets/global-foo.js b/test-runner/test/assets/global-foo.js deleted file mode 100644 index ccfbc9e3784c7..0000000000000 --- a/test-runner/test/assets/global-foo.js +++ /dev/null @@ -1,4 +0,0 @@ -global.foo = true; -module.exports = { - abc: 123 -}; \ No newline at end of file diff --git a/test-runner/test/assets/nested-skip.js b/test-runner/test/assets/nested-skip.js deleted file mode 100644 index acbe569d226fa..0000000000000 --- a/test-runner/test/assets/nested-skip.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../'); - -describe('skipped', suite => { - suite.skip(true); -}, () => { - it('succeeds',() => { - expect(1 + 1).toBe(2); - }); -}); diff --git a/test-runner/test/assets/one-failure.js b/test-runner/test/assets/one-failure.js deleted file mode 100644 index 2261932954182..0000000000000 --- a/test-runner/test/assets/one-failure.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../'); - -it('fails', () => { - expect(1 + 1).toBe(7); -}); diff --git a/test-runner/test/assets/one-success.js b/test-runner/test/assets/one-success.js deleted file mode 100644 index e2b85cdee3581..0000000000000 --- a/test-runner/test/assets/one-success.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../'); - -it('succeeds', () => { - expect(1 + 1).toBe(2); -}); diff --git a/test-runner/test/assets/one-timeout.js b/test-runner/test/assets/one-timeout.js deleted file mode 100644 index 2b0f4cebe5e45..0000000000000 --- a/test-runner/test/assets/one-timeout.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../'); - -it('timeout', async () => { - await new Promise(f => setTimeout(f, 10000)); -}); diff --git a/test-runner/test/assets/retry-failures.js b/test-runner/test/assets/retry-failures.js deleted file mode 100644 index f29af760498df..0000000000000 --- a/test-runner/test/assets/retry-failures.js +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const fs = require('fs'); -const path = require('path'); - -it('flake', async ({}) => { - try { - fs.readFileSync(path.join(__dirname, '..', 'test-results', 'retry-failures.txt')); - } catch (e) { - // First time this fails. - fs.writeFileSync(path.join(__dirname, '..', 'test-results', 'retry-failures.txt'), 'TRUE'); - expect(true).toBe(false); - } -}); diff --git a/test-runner/test/assets/stdio.js b/test-runner/test/assets/stdio.js deleted file mode 100644 index a7c33c7aad52e..0000000000000 --- a/test-runner/test/assets/stdio.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -require('../../..'); - -it('stdio', () => { - process.stdout.write('stdout text'); - process.stdout.write(Buffer.from('stdout buffer')); - process.stderr.write('stderr text'); - process.stderr.write(Buffer.from('stderr buffer')); -}); diff --git a/test-runner/test/assets/suite-error.js b/test-runner/test/assets/suite-error.js deleted file mode 100644 index cf244a37f0d7e..0000000000000 --- a/test-runner/test/assets/suite-error.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -const { parameters } = require('../../'); - -if (typeof parameters.parallelIndex === 'number') - throw new Error('Suite error'); - -it('passes',() => { - expect(1 + 1).toBe(2); -}); diff --git a/test-runner/test/assets/test-data-visible-in-fixture.js b/test-runner/test/assets/test-data-visible-in-fixture.js deleted file mode 100644 index 41d56b1c02105..0000000000000 --- a/test-runner/test/assets/test-data-visible-in-fixture.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { registerFixture } = require('../../'); -const fs = require('fs'); -const path = require('path'); - -registerFixture('postProcess', async ({}, runTest, info) => { - await runTest(''); - info.result.data['myname'] = 'myvalue'; -}); - -it('ensure fixture handles test error', async ({ postProcess }) => { - console.log('console.log'); - console.error('console.error'); - expect(true).toBe(false); -}); diff --git a/test-runner/test/assets/test-error-visible-in-fixture.js b/test-runner/test/assets/test-error-visible-in-fixture.js deleted file mode 100644 index 45228e67e023e..0000000000000 --- a/test-runner/test/assets/test-error-visible-in-fixture.js +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { registerFixture } = require('../../'); -const fs = require('fs'); -const path = require('path'); - -registerFixture('postProcess', async ({}, runTest, info) => { - await runTest(''); - const { config, result } = info; - fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(result.error, undefined, 2)); -}); - -it('ensure fixture handles test error', async ({ postProcess }) => { - expect(true).toBe(false); -}); diff --git a/test-runner/test/assets/typescript.ts b/test-runner/test/assets/typescript.ts deleted file mode 100644 index e0bd3b3447e4a..0000000000000 --- a/test-runner/test/assets/typescript.ts +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import '../../'; -import './global-foo'; - -it('should find global foo', () => { - expect(global['foo']).toBe(true); -}); - -it('should work with type annotations', () => { - const x: number = 5; - expect(x).toBe(5); -}); \ No newline at end of file diff --git a/test-runner/test/assets/unexpected-pass.js b/test-runner/test/assets/unexpected-pass.js deleted file mode 100644 index 0aaf70c22893c..0000000000000 --- a/test-runner/test/assets/unexpected-pass.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../'); - -it('succeeds', test => test.fail(), () => { - expect(1 + 1).toBe(2); -}); diff --git a/test-runner/test/assets/unhandled-rejection.js b/test-runner/test/assets/unhandled-rejection.js deleted file mode 100644 index 8bf0d72941234..0000000000000 --- a/test-runner/test/assets/unhandled-rejection.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -require('../../'); - -it('unhandled rejection', async () => { - setTimeout(() => { - throw new Error('Unhandled rejection in the test'); - }); - await new Promise(f => setTimeout(f, 20)); -}); diff --git a/test-runner/test/assets/worker-fixture-error.js b/test-runner/test/assets/worker-fixture-error.js deleted file mode 100644 index d9130b6ee60d9..0000000000000 --- a/test-runner/test/assets/worker-fixture-error.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { registerWorkerFixture } = require('../../'); - -registerWorkerFixture('failure', async ({}, runTest) => { - throw new Error('Worker failed'); -}); - -it('fails', async({failure}) => { -}); diff --git a/test-runner/test/assets/worker-fixture-timeout.js b/test-runner/test/assets/worker-fixture-timeout.js deleted file mode 100644 index 2aabcae9fb1dc..0000000000000 --- a/test-runner/test/assets/worker-fixture-timeout.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { registerWorkerFixture } = require('../../'); - -registerWorkerFixture('timeout', async ({}, runTest) => { -}); - -it('fails', async({timeout}) => { -}); diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts deleted file mode 100644 index a243b0440ac4f..0000000000000 --- a/test-runner/test/exit-code.spec.ts +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import colors from 'colors/safe'; -import { spawnSync } from 'child_process'; -import * as fs from 'fs'; -import * as path from 'path'; -import rimraf from 'rimraf'; -import { promisify } from 'util'; -import '../lib'; - -const removeFolderAsync = promisify(rimraf); - -it('should fail', async () => { - const result = await runTest('one-failure.js'); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); -}); - -it('should timeout', async () => { - const { exitCode, passed, failed, timedOut } = await runTest('one-timeout.js', { timeout: 100 }); - expect(exitCode).toBe(1); - expect(passed).toBe(0); - expect(failed).toBe(0); - expect(timedOut).toBe(1); -}); - -it('should succeed', async () => { - const result = await runTest('one-success.js'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); -}); - -it('should access error in fixture', async () => { - const result = await runTest('test-error-visible-in-fixture.js'); - expect(result.exitCode).toBe(1); - const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'test-error-visible-in-fixture.txt')).toString()); - expect(data.message).toContain('Object.is equality'); -}); - -it('should access data in fixture', async () => { - const { exitCode, report } = await runTest('test-data-visible-in-fixture.js'); - expect(exitCode).toBe(1); - const testResult = report.suites[0].tests[0].results[0]; - expect(testResult.data).toEqual({ 'myname': 'myvalue' }); - expect(testResult.stdout).toEqual([{ text: 'console.log\n' }]); - expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]); -}); - -it('should handle fixture timeout', async () => { - const { exitCode, output, failed, timedOut } = await runTest('fixture-timeout.js', { timeout: 500 }); - expect(exitCode).toBe(1); - expect(output).toContain('Timeout of 500ms'); - expect(failed).toBe(1); - expect(timedOut).toBe(1); -}); - -it('should handle worker fixture timeout', async () => { - const result = await runTest('worker-fixture-timeout.js', { timeout: 500 }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain('Timeout of 500ms'); -}); - -it('should handle worker fixture error', async () => { - const result = await runTest('worker-fixture-error.js'); - expect(result.exitCode).toBe(1); - expect(result.output).toContain('Worker failed'); -}); - -it('should collect stdio', async () => { - const { exitCode, report } = await runTest('stdio.js'); - expect(exitCode).toBe(0); - const testResult = report.suites[0].tests[0].results[0]; - const { stdout, stderr } = testResult; - expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]); - expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]); -}); - -it('should work with typescript', async () => { - const result = await runTest('typescript.ts'); - expect(result.exitCode).toBe(0); -}); - -it('should retry failures', async () => { - const result = await runTest('retry-failures.js', { retries: 1 }); - expect(result.exitCode).toBe(1); - expect(result.flaky).toBe(1); -}); - -it('should retry timeout', async () => { - const { exitCode, passed, failed, timedOut, output } = await runTest('one-timeout.js', { timeout: 100, retries: 2 }); - expect(exitCode).toBe(1); - expect(passed).toBe(0); - expect(failed).toBe(0); - expect(timedOut).toBe(1); - expect(output.split('\n')[0]).toBe(colors.red('T').repeat(3)); -}); - -it('should repeat each', async () => { - const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 }); - expect(exitCode).toBe(0); - expect(report.suites.length).toBe(3); - for (const suite of report.suites) - expect(suite.tests.length).toBe(1); -}); - -it('should report suite errors', async () => { - const { exitCode, failed, output } = await runTest('suite-error.js'); - expect(exitCode).toBe(1); - expect(failed).toBe(1); - expect(output).toContain('Suite error'); -}); - -it('should allow flaky', async () => { - const result = await runTest('allow-flaky.js', { retries: 1 }); - expect(result.exitCode).toBe(0); - expect(result.flaky).toBe(1); -}); - -it('should fail on unexpected pass', async () => { - const { exitCode, failed, output } = await runTest('unexpected-pass.js'); - expect(exitCode).toBe(1); - expect(failed).toBe(1); - expect(output).toContain('passed unexpectedly'); -}); - -it('should fail on unexpected pass with retries', async () => { - const { exitCode, failed, output } = await runTest('unexpected-pass.js', { retries: 1 }); - expect(exitCode).toBe(1); - expect(failed).toBe(1); - expect(output).toContain('passed unexpectedly'); -}); - -it('should not retry unexpected pass', async () => { - const { exitCode, passed, failed, output } = await runTest('unexpected-pass.js', { retries: 2 }); - expect(exitCode).toBe(1); - expect(passed).toBe(0); - expect(failed).toBe(1); - expect(output.split('\n')[0]).toBe(colors.red('P')); -}); - -it('should not retry expected failure', async () => { - const { exitCode, passed, failed, output } = await runTest('expected-failure.js', { retries: 2 }); - expect(exitCode).toBe(0); - expect(passed).toBe(2); - expect(failed).toBe(0); - expect(output.split('\n')[0]).toBe(colors.green('f') + colors.green('·')); -}); - -it('should respect nested skip', async () => { - const { exitCode, passed, failed, skipped } = await runTest('nested-skip.js'); - expect(exitCode).toBe(0); - expect(passed).toBe(0); - expect(failed).toBe(0); - expect(skipped).toBe(1); -}); - -it('should retry unhandled rejection', async () => { - const result = await runTest('unhandled-rejection.js', { retries: 2 }); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); - expect(result.output.split('\n')[0]).toBe(colors.red('F').repeat(3)); - expect(result.output).toContain('Unhandled rejection'); -}); - -it('should respect global timeout', async () => { - const { exitCode, output } = await runTest('one-timeout.js', { 'timeout': 100000, 'global-timeout': 500 }); - expect(exitCode).toBe(1); - expect(output).toContain('Timed out waiting 0.5s for the entire test run'); -}); - -async function runTest(filePath: string, params: any = {}) { - const outputDir = path.join(__dirname, 'test-results'); - const reportFile = path.join(outputDir, 'results.json'); - await removeFolderAsync(outputDir).catch(e => {}); - - const { output, status } = spawnSync('node', [ - path.join(__dirname, '..', 'cli.js'), - path.join(__dirname, 'assets', filePath), - '--output=' + outputDir, - '--reporter=dot,json', - ...Object.keys(params).map(key => `--${key}=${params[key]}`) - ], { - env: { - ...process.env, - PWRUNNER_JSON_REPORT: reportFile, - } - }); - const passed = (/(\d+) passed/.exec(output.toString()) || [])[1]; - const failed = (/(\d+) failed/.exec(output.toString()) || [])[1]; - const timedOut = (/(\d+) timed out/.exec(output.toString()) || [])[1]; - const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1]; - const skipped = (/(\d+) skipped/.exec(output.toString()) || [])[1]; - const report = JSON.parse(fs.readFileSync(reportFile).toString()); - let outputStr = output.toString(); - outputStr = outputStr.substring(1, outputStr.length - 1); - return { - exitCode: status, - output: outputStr, - passed: parseInt(passed, 10), - failed: parseInt(failed || '0', 10), - timedOut: parseInt(timedOut || '0', 10), - flaky: parseInt(flaky || '0', 10), - skipped: parseInt(skipped || '0', 10), - report - }; -} diff --git a/test-runner/tsconfig.json b/test-runner/tsconfig.json deleted file mode 100644 index bdffe9953081c..0000000000000 --- a/test-runner/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "compilerOptions": { - "moduleResolution": "node", - "target": "ESNext", - "module": "commonjs", - "lib": ["esnext", "dom", "DOM.Iterable"], - "sourceMap": true, - "rootDir": "./src", - "outDir": "./lib", - "esModuleInterop": true, - "allowSyntheticDefaultImports": true, - "declaration": true, - "allowJs": true, - "checkJs": true, - "declarationMap": true - }, - "compileOnSave": true, - "include": ["src"], - "exclude": ["node_modules"] -} \ No newline at end of file diff --git a/test/chromium/oopif.spec.ts b/test/chromium/oopif.spec.ts index 786698fc8044c..e79cee1f3a273 100644 --- a/test/chromium/oopif.spec.ts +++ b/test/chromium/oopif.spec.ts @@ -15,7 +15,7 @@ */ import { options } from '../playwright.fixtures'; -import { registerWorkerFixture } from '../../test-runner'; +import { registerWorkerFixture } from '@playwright/test-runner'; registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => { const browser = await browserType.launch({ diff --git a/test/chromium/tracing.spec.ts b/test/chromium/tracing.spec.ts index eea8a5eea0e4b..9faec67c240a0 100644 --- a/test/chromium/tracing.spec.ts +++ b/test/chromium/tracing.spec.ts @@ -15,7 +15,7 @@ */ import { options } from '../playwright.fixtures'; -import { registerFixture } from '../../test-runner'; +import { registerFixture } from '@playwright/test-runner'; import fs from 'fs'; import path from 'path'; diff --git a/test/downloads-path.spec.ts b/test/downloads-path.spec.ts index ef4f215b1f8fe..5796f5fd7f4b5 100644 --- a/test/downloads-path.spec.ts +++ b/test/downloads-path.spec.ts @@ -17,7 +17,7 @@ import './playwright.fixtures'; -import { registerFixture } from '../test-runner'; +import { registerFixture } from '@playwright/test-runner'; import path from 'path'; import fs from 'fs'; diff --git a/test/electron/electron.fixture.ts b/test/electron/electron.fixture.ts index 472861bd27552..6a4c4e8046186 100644 --- a/test/electron/electron.fixture.ts +++ b/test/electron/electron.fixture.ts @@ -15,7 +15,7 @@ */ import '../playwright.fixtures'; -import { registerFixture } from '../../test-runner'; +import { registerFixture } from '@playwright/test-runner'; import type {ElectronApplication, ElectronLauncher, ElectronPage} from '../../electron-types'; import path from 'path'; diff --git a/test/playwright.fixtures.ts b/test/playwright.fixtures.ts index cf987ac6fc424..1635894ad6859 100644 --- a/test/playwright.fixtures.ts +++ b/test/playwright.fixtures.ts @@ -23,7 +23,7 @@ import { TestServer } from '../utils/testserver'; import { Connection } from '../lib/client/connection'; import { Transport } from '../lib/protocol/transport'; import { installCoverageHooks } from './coverage'; -import { parameters, registerFixture, registerWorkerFixture } from '../test-runner'; +import { parameters, registerFixture, registerWorkerFixture } from '@playwright/test-runner'; import { mkdtempAsync, removeFolderAsync } from './utils'; export const options = { diff --git a/test/proxy.spec.ts b/test/proxy.spec.ts index 389bf40c1df30..c2b5dea757587 100644 --- a/test/proxy.spec.ts +++ b/test/proxy.spec.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { parameters } from '../test-runner'; +import { parameters } from '@playwright/test-runner'; import { options } from './playwright.fixtures'; import socks from 'socksv5'; diff --git a/test/remoteServer.fixture.ts b/test/remoteServer.fixture.ts index 6cb5f39d36606..327a3a7eea27e 100644 --- a/test/remoteServer.fixture.ts +++ b/test/remoteServer.fixture.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { registerFixture } from '../test-runner/lib'; +import { registerFixture } from '@playwright/test-runner'; import path from 'path'; import { spawn } from 'child_process'; diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts index 05455304e63b9..8a06fbdded5e9 100644 --- a/test/screencast.spec.ts +++ b/test/screencast.spec.ts @@ -15,7 +15,7 @@ */ import { options } from './playwright.fixtures'; -import { registerFixture } from '../test-runner'; +import { registerFixture } from '@playwright/test-runner'; import type { Page } from '..'; import fs from 'fs'; diff --git a/test/test-runner-helper.ts b/test/test-runner-helper.ts index 1f8d8de441544..0a73f875280c7 100644 --- a/test/test-runner-helper.ts +++ b/test/test-runner-helper.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { registerFixture } from '../test-runner'; +import { registerFixture } from '@playwright/test-runner'; declare global { interface TestState { diff --git a/test/test-runner-overrides-1.spec.ts b/test/test-runner-overrides-1.spec.ts index 1e9fafddd28fb..c8253014c4c81 100644 --- a/test/test-runner-overrides-1.spec.ts +++ b/test/test-runner-overrides-1.spec.ts @@ -15,7 +15,7 @@ */ import './test-runner-helper'; -import { registerFixture } from '../test-runner'; +import { registerFixture } from '@playwright/test-runner'; registerFixture('helperFixture', async ({}, test) => { await test('helperFixture - overridden'); diff --git a/test/test-runner.spec.ts b/test/test-runner.spec.ts index d11bb7116a9e0..08624e3ac9222 100644 --- a/test/test-runner.spec.ts +++ b/test/test-runner.spec.ts @@ -15,7 +15,7 @@ */ import './test-runner-helper'; -import { registerFixture } from '../test-runner'; +import { registerFixture } from '@playwright/test-runner'; declare global { interface TestState { diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index 0211afe99317f..b9b8b73cf7745 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -20,7 +20,7 @@ const checkPublicAPI = require('..'); const Source = require('../../Source'); const mdBuilder = require('../MDBuilder'); const jsBuilder = require('../JSBuilder'); -const { registerWorkerFixture } = require('../../../../test-runner'); +const { registerWorkerFixture } = require('@playwright/test-runner'); registerWorkerFixture('page', async({}, test) => { const browser = await playwright.chromium.launch(); diff --git a/utils/watch.js b/utils/watch.js index 0c4c3b1406814..5ea01ad09b4f2 100644 --- a/utils/watch.js +++ b/utils/watch.js @@ -21,7 +21,6 @@ const fs = require('fs'); const spawns = [ child_process.spawn('node', [path.join(__dirname, 'runWebpack.js'), '--mode="development"', '--watch', '--silent'], { stdio: 'inherit', shell: true }), child_process.spawn('npx', ['tsc', '-w', '--preserveWatchOutput', '-p', path.join(__dirname, '..')], { stdio: 'inherit', shell: true }), - child_process.spawn('npx', ['tsc', '-w', '--preserveWatchOutput', '-p', path.join(__dirname, '..', 'test-runner')], { stdio: 'inherit', shell: true }), ]; process.on('exit', () => spawns.forEach(s => s.kill()));