From b342adbee4581cd6f9bf1cad3eb97cf8ed9cec32 Mon Sep 17 00:00:00 2001 From: Alex Kamaev Date: Thu, 29 Aug 2019 17:57:47 +0300 Subject: [PATCH] fix quarantine for live mode (closes #4093) --- src/live/test-run-controller.js | 23 +++++-- src/runner/test-run-controller.js | 6 ++ test/functional/fixtures/live/test-helper.js | 8 ++- test/functional/fixtures/live/test.js | 66 +++++++++++++++---- .../live/testcafe-fixtures/quarantine.js | 14 ++++ .../{index-test.js => smoke.js} | 0 6 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 test/functional/fixtures/live/testcafe-fixtures/quarantine.js rename test/functional/fixtures/live/testcafe-fixtures/{index-test.js => smoke.js} (100%) diff --git a/src/live/test-run-controller.js b/src/live/test-run-controller.js index 41126f92a74..0dc43f0dfe9 100644 --- a/src/live/test-run-controller.js +++ b/src/live/test-run-controller.js @@ -1,5 +1,5 @@ import EventEmitter from 'events'; -import noop from 'lodash'; +import { noop, uniq } from 'lodash'; import { TestRunCtorFactory } from './test-run'; import TEST_RUN_STATE from './test-run-state'; @@ -66,7 +66,6 @@ class LiveModeTestRunController extends EventEmitter { } _onTestRunCreated (testRun) { - this.allTestsCompletePromise = new Promise(resolve => { this.completeAllRunningTests = resolve; }); @@ -81,25 +80,28 @@ class LiveModeTestRunController extends EventEmitter { _onTestRunDone (testRun) { testRun.state = TEST_RUN_STATE.done; - const hasRunningTests = this._getTestRuns().some(t => t.state !== TEST_RUN_STATE.done); + const testWillBeRestarted = !this.isTestFinished(testRun); + const hasRunningTestsInOtherBrowsers = this._getTestRuns().some(t => t.state !== TEST_RUN_STATE.done); - if (!hasRunningTests) + if (!hasRunningTestsInOtherBrowsers && !testWillBeRestarted) this.emit('all-tests-complete'); const browserTestRuns = this.testRuns[testRun.browserConnection.id]; + const tests = uniq(browserTestRuns.map(t => t.test)); testRun.readyToNextPromise = new Promise(resolve => { testRun.setReadyToNext = resolve; }); + const isLastTestRun = tests.length >= this.expectedTestCount; - if (browserTestRuns.length < this.expectedTestCount || browserTestRuns.some(t => t.state !== TEST_RUN_STATE.done)) + if (testWillBeRestarted || !isLastTestRun) return Promise.resolve(); return new Promise(resolve => { testRun.finish = () => { testRun.finish = null; - testRun.state = TEST_RUN_STATE.done; + resolve(); }; }); @@ -108,6 +110,15 @@ class LiveModeTestRunController extends EventEmitter { _onTestRunReadyToNext (testRun) { testRun.setReadyToNext(); } + + isTestFinished (testRun) { + const { quarantine, errs } = testRun; + + if (!quarantine) + return true; + + return quarantine.isFirstAttemptSuccessfull(errs) || quarantine.isThresholdReached(errs); + } } export default LiveModeTestRunController; diff --git a/src/runner/test-run-controller.js b/src/runner/test-run-controller.js index ed5d3dca665..26fe6c59f32 100644 --- a/src/runner/test-run-controller.js +++ b/src/runner/test-run-controller.js @@ -32,6 +32,12 @@ class Quarantine { return failedThresholdReached || passedThresholdReached; } + isFirstAttemptSuccessfull (extraErrors) { + const { failedTimes, passedTimes } = this._getAttemptsResult(extraErrors); + + return failedTimes === 0 && passedTimes > 0; + } + _getAttemptsResult (extraErrors) { let failedTimes = this.getFailedAttempts().length; let passedTimes = this.getPassedAttempts().length; diff --git a/test/functional/fixtures/live/test-helper.js b/test/functional/fixtures/live/test-helper.js index 22129defcb0..b105b26fa3d 100644 --- a/test/functional/fixtures/live/test-helper.js +++ b/test/functional/fixtures/live/test-helper.js @@ -6,7 +6,9 @@ class TestCompleteWatcher extends EventEmitter { } module.exports = { - watcher: new TestCompleteWatcher(), - counter: 0, - testCount: 10 + watcher: new TestCompleteWatcher(), + counter: 0, + attempts: 0, + testCount: 10, + quarantineThreshold: 3 }; diff --git a/test/functional/fixtures/live/test.js b/test/functional/fixtures/live/test.js index 7f5f49dbfac..cb541a59e45 100644 --- a/test/functional/fixtures/live/test.js +++ b/test/functional/fixtures/live/test.js @@ -1,15 +1,38 @@ +const { noop } = require('lodash'); const createTestCafe = require('../../../../lib'); const config = require('../../config'); const path = require('path'); const { expect } = require('chai'); const helper = require('./test-helper'); +const browsers = ['chrome', 'firefox']; +let cafe = null; + +function createRunner (tc, src) { + const runner = tc.createLiveModeRunner(); + + runner.controller._listenKeyPress = () => { }; + runner.controller._initFileWatching = () => { }; + runner.controller.dispose = () => { }; + + src = path.join(__dirname, src); + + return runner + .src(src) + .browsers(browsers) + .reporter(() => { + return { + reportTaskStart: noop, + reportTaskDone: noop, + reportTestDone: noop, + reportFixtureStart: noop + }; + }); +} if (config.useLocalBrowsers && !config.useHeadlessBrowsers) { - describe('Live smoke', () => { + describe.only('Live Mode tests', () => { it('Live smoke', () => { - let cafe = null; - const browsers = ['chrome', 'firefox']; const runCount = 2; return createTestCafe('127.0.0.1', 1335, 1336) @@ -17,12 +40,7 @@ if (config.useLocalBrowsers && !config.useHeadlessBrowsers) { cafe = tc; }) .then(() => { - const runner = cafe.createLiveModeRunner(); - const fixturePath = path.join(__dirname, '/testcafe-fixtures/index-test.js'); - - runner.controller._listenKeyPress = () => { }; - runner.controller._initFileWatching = () => { }; - runner.controller.dispose = () => { }; + const runner = createRunner(cafe, '/testcafe-fixtures/smoke.js'); helper.watcher.once('test-complete', () => { setTimeout(() => { @@ -33,10 +51,7 @@ if (config.useLocalBrowsers && !config.useHeadlessBrowsers) { }, 1000); }); - return runner - .src(fixturePath) - .browsers(browsers) - .run(); + return runner.run(); }) .then(() => { expect(helper.counter).eql(browsers.length * helper.testCount * runCount); @@ -44,5 +59,30 @@ if (config.useLocalBrowsers && !config.useHeadlessBrowsers) { return cafe.close(); }); }); + + it('Live quarantine', () => { + return createTestCafe('127.0.0.1', 1335, 1336) + .then(tc => { + cafe = tc; + }) + .then(() => { + const runner = createRunner(cafe, '/testcafe-fixtures/quarantine.js'); + + helper.watcher.once('test-complete', () => { + setTimeout(() => { + runner.exit(); + }, 1000); + }); + + return runner.run({ + quarantineMode: true + }); + }) + .then(() => { + expect(helper.attempts).eql(browsers.length * helper.quarantineThreshold); + + return cafe.close(); + }); + }); }); } diff --git a/test/functional/fixtures/live/testcafe-fixtures/quarantine.js b/test/functional/fixtures/live/testcafe-fixtures/quarantine.js new file mode 100644 index 00000000000..5cf09fe6740 --- /dev/null +++ b/test/functional/fixtures/live/testcafe-fixtures/quarantine.js @@ -0,0 +1,14 @@ +import helper from '../test-helper'; + +fixture `Live` + .page `../pages/index.html` + .afterEach(() => { + helper.attempts++; + }) + .after(() => { + helper.watcher.emit('test-complete'); + }); + +test('quarantine', async () => { + throw new Error('error'); +}); diff --git a/test/functional/fixtures/live/testcafe-fixtures/index-test.js b/test/functional/fixtures/live/testcafe-fixtures/smoke.js similarity index 100% rename from test/functional/fixtures/live/testcafe-fixtures/index-test.js rename to test/functional/fixtures/live/testcafe-fixtures/smoke.js