From 3bdf0e804aba74e77cf29b7d4e6bcd2aa69d427a Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 24 Aug 2020 14:55:48 -0700 Subject: [PATCH] fix(testrunner): pass error into test fixtures (#3605) --- .gitignore | 1 + test-runner/src/builtin.fixtures.ts | 4 +- test-runner/src/fixtures.ts | 5 ++- test-runner/src/index.ts | 2 +- test-runner/src/test.ts | 24 +++++++++++ test-runner/src/testRunner.ts | 26 +----------- .../assets/test-error-visible-in-fixture.js | 28 +++++++++++++ test-runner/test/exit-code.spec.ts | 40 ++++++++++++++----- 8 files changed, 91 insertions(+), 39 deletions(-) create mode 100644 test-runner/test/assets/test-error-visible-in-fixture.js diff --git a/.gitignore b/.gitignore index 2289dac322220..b160baf88f0f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /node_modules/ /test-results/ +/test-runner/test/test-results/ /test/coverage-report /test/test-user-data-dir* .local-browsers/ diff --git a/test-runner/src/builtin.fixtures.ts b/test-runner/src/builtin.fixtures.ts index f3dd317981d7f..27e47d3f9097c 100644 --- a/test-runner/src/builtin.fixtures.ts +++ b/test-runner/src/builtin.fixtures.ts @@ -16,10 +16,10 @@ import os from 'os'; import path from 'path'; -import {promisify} from 'util'; +import { promisify } from 'util'; import fs from 'fs'; import rimraf from 'rimraf'; -import {registerFixture} from './fixtures'; +import { registerFixture } from './fixtures'; declare global { diff --git a/test-runner/src/fixtures.ts b/test-runner/src/fixtures.ts index b83e001506fb5..fb4e044de50cb 100644 --- a/test-runner/src/fixtures.ts +++ b/test-runner/src/fixtures.ts @@ -15,7 +15,7 @@ */ import debug from 'debug'; -import { Test } from './test'; +import { Test, serializeError } from './test'; type Scope = 'test' | 'worker'; @@ -159,6 +159,9 @@ export class FixturePool { return async() => { try { await this.resolveParametersAndRun(callback, timeout, config, test); + } catch (e) { + test.error = serializeError(e); + throw e; } finally { await this.teardownScope('test'); } diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index 2dcf62151306b..c6c2c6191cdde 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -81,7 +81,7 @@ export async function runTests(config: RunnerConfig, suite: Suite, reporter: Rep // Trial run does not need many workers, use one. const jobs = (config.trialRun || config.debug) ? 1 : config.jobs; const runner = new Runner(suite, { ...config, jobs }, reporter); - + fs.mkdirSync(config.outputDir, { recursive: true }); try { for (const f of beforeFunctions) await f(); diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts index a5fb5928120bb..cc3ee9293d5b8 100644 --- a/test-runner/src/test.ts +++ b/test-runner/src/test.ts @@ -164,3 +164,27 @@ export function serializeConfiguration(configuration: Configuration): string { tokens.push(`${name}=${value}`); return tokens.join(', '); } + +export function serializeError(error: Error): any { + if (error instanceof Error) { + return { + message: error.message, + stack: error.stack + } + } + return trimCycles(error); +} + +function trimCycles(obj: any): any { + const cache = new Set(); + return JSON.parse( + JSON.stringify(obj, function(key, value) { + if (typeof value === 'object' && value !== null) { + if (cache.has(value)) + return '' + value; + cache.add(value); + } + return value; + }) + ); +} diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 762a003b41436..722f89ed24112 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -17,7 +17,7 @@ import { FixturePool, rerunRegistrations, setParameters } from './fixtures'; import { EventEmitter } from 'events'; import { setCurrentTestFile } from './expect'; -import { Test, Suite, Configuration } from './test'; +import { Test, Suite, Configuration, serializeError } from './test'; import { spec } from './spec'; import { RunnerConfig } from './runnerConfig'; @@ -163,27 +163,3 @@ export class TestRunner extends EventEmitter { }; } } - -function trimCycles(obj: any): any { - const cache = new Set(); - return JSON.parse( - JSON.stringify(obj, function(key, value) { - if (typeof value === 'object' && value !== null) { - if (cache.has(value)) - return '' + value; - cache.add(value); - } - return value; - }) - ); -} - -function serializeError(error: Error): any { - if (error instanceof Error) { - return { - message: error.message, - stack: error.stack - } - } - return trimCycles(error); -} diff --git a/test-runner/test/assets/test-error-visible-in-fixture.js b/test-runner/test/assets/test-error-visible-in-fixture.js new file mode 100644 index 0000000000000..7a63cd83c8d67 --- /dev/null +++ b/test-runner/test/assets/test-error-visible-in-fixture.js @@ -0,0 +1,28 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { registerFixture } = require('../../'); +const fs = require('fs'); +const path = require('path'); + +registerFixture('postProcess', async ({}, runTest, config, test) => { + await runTest(''); + fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(test.error, undefined, 2)); +}); + +it('ensure fixture handles test error', async ({ postProcess }) => { + expect(true).toBe(false); +}); diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts index bf6e5d1c6f2c0..92b84f46c1fcd 100644 --- a/test-runner/test/exit-code.spec.ts +++ b/test-runner/test/exit-code.spec.ts @@ -13,32 +13,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import "../lib" -import { spawnSync } from "child_process"; -import path from 'path'; + +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import rimraf from 'rimraf'; +import { promisify } from 'util'; +import '../lib'; + +const removeFolderAsync = promisify(rimraf); it('should fail', async() => { - const result = runTest('one-failure.js'); + const result = await runTest('one-failure.js'); expect(result.exitCode).toBe(1); expect(result.passed).toBe(0); expect(result.failed).toBe(1); }); it('should succeed', async() => { - const result = runTest('one-success.js'); + const result = await runTest('one-success.js'); expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); expect(result.failed).toBe(0); }); -function runTest(filePath: string) { - const {output, status} = spawnSync('node', [path.join(__dirname, '..', 'cli.js'), path.join(__dirname, 'assets', filePath)]); - const passed = (/ (\d+) passed/.exec(output.toString()) || [])[1]; - const failed = (/ (\d+) failed/.exec(output.toString()) || [])[1]; +it('should access error in fixture', async() => { + const result = await runTest('test-error-visible-in-fixture.js'); + expect(result.exitCode).toBe(1); + const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'test-error-visible-in-fixture.txt')).toString()); + expect(data.message).toContain('Object.is equality'); +}); + +async function runTest(filePath: string) { + const outputDir = path.join(__dirname, 'test-results') + await removeFolderAsync(outputDir).catch(e => {}); + + const { output, status } = spawnSync('node', [ + path.join(__dirname, '..', 'cli.js'), + path.join(__dirname, 'assets', filePath), + '--output=' + outputDir + ]); + const passed = (/(\d+) passed/.exec(output.toString()) || [])[1]; + const failed = (/(\d+) failed/.exec(output.toString()) || [])[1]; return { exitCode: status, output, passed: parseInt(passed), failed: parseInt(failed || '0') } -} \ No newline at end of file +}