feat(core): simple mutation switching process (#2269)

Implement a simple mutation testing process using mutation switching. It represents a new way of running mutation testing, where we place all mutants in the code at the same time. This is only the first step and lacks support for more advanced use cases.

This is a non-exhaustive list of changes:
* Add `Location` to the `Mutant` API, for ease of reporting
* Implement `TestRunner2` factory
* Revamped the way the main Stryker class works. All of the internals are moved to separate `Executor` classes. An `Executor` is responsible for _managing_ a part of the process. I think this helps enormously to understand the general process of mutation testing.
  * `PrepareExecutor`: Responsible to read configuration and input files from disk.
  * `MutantInstrumenterExecutor`: Responsible for instrumenting the source code and generating mutants
  * `DryRunExecutor`: Responsible for performing the initial test run.
  * `MutationTestExecutor`: Responsible for running actual mutation testing.
* Revamped the `Sandbox` class. It is now only responsible for filling a sandbox with files. It no longer manages mutants inside the sandbox or the test runner instance. 
* Revamped the `SandboxPool` -> `TestRunnerPool`, which now manages only test runner instances.
   * An improvement I've managed to sneak in, is that the test runner process from the dry run is now reused in the mutation test run. Not a huge deal, but a nice bonus.
* Removed a bunch of dead code related to the `TestFramework` and `Mutator` plugins, see breaking changes
* Re-enabled 2 simple e2e tests 🎉 

BREAKING CHANGES:

Most of these changes are related to internals of Stryker plugins.

* Transpilers plugins are no longer supported. Transpiler-like functionality will be possible with a build command and checker API. More on those in future PR's
* Custom mutators are no longer supported, Stryker now brings its own mutator in the form of `@stryker-mutator/instrumenter`. It uses babel and an HTML parser under the hood so it has support for all the usual suspects.
* Test Framework plugins are no longer supported. Test runners are supposed to bring their own support for test frameworks they might want to support. This has no consequences for the list of officially supported test runners.
* Test runner v1 API is no longer supported. Please move to the new Test Runner API.

.github/workflows/ci.yml

@@ -29,19 +29,19 @@ jobs:
     - name: Build & lint & test
       run: npm run all
 
-  # e2e:
-  #   runs-on: ${{ matrix.os }}
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       os: ['ubuntu-latest', 'windows-latest']
+  e2e:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest', 'windows-latest']
 
-  #   steps:
-  #   - uses: actions/checkout@v1
-  #   - name: Install dependencies
-  #     run: npm install
-  #   - name: Build packages
-  #     run: npm run build
-  #   - name: Run e2e tests
-  #     run: npm run e2e
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install dependencies
+      run: npm install
+    - name: Build packages
+      run: npm run build
+    - name: Run e2e tests
+      run: npm run e2e
 

e2e/package.json

@@ -45,6 +45,7 @@
   "localDependencies": {
     "grunt-stryker": "../packages/grunt-stryker",
     "@stryker-mutator/core": "../packages/core",
+    "@stryker-mutator/instrumenter": "../packages/instrumenter",
     "@stryker-mutator/api": "../packages/api",
     "@stryker-mutator/babel-transpiler": "../packages/babel-transpiler",
     "@stryker-mutator/jasmine-framework": "../packages/jasmine-framework",

e2e/tasks/run-e2e-tests.ts

@@ -8,12 +8,19 @@ import { tap, mergeAll, map, filter } from 'rxjs/operators';
 
 const testRootDir = path.resolve(__dirname, '..', 'test');
 
+const mutationSwitchingTempWhiteList = [
+  'jasmine-jasmine',
+  'karma-mocha',
+  'karma-jasmine',
+]
+
 function runE2eTests() {
   const testDirs = fs.readdirSync(testRootDir);
 
   // Create test$, an observable of test runs
   const test$ = from(testDirs).pipe(
     filter(dir => fs.statSync(path.join(testRootDir, dir)).isDirectory()),
+    filter(dir => mutationSwitchingTempWhiteList.includes(dir)),
     map(testDir => defer(() => runTest(testDir)))
   );
 

e2e/test/angular-project/package.json

@@ -42,6 +42,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/karma-runner": "../../../packages/karma-runner",
     "@stryker-mutator/typescript": "../../../packages/typescript",
     "@stryker-mutator/util": "../../../packages/util"

e2e/test/jasmine-jasmine/stryker.conf.js

@@ -6,8 +6,9 @@ module.exports = function (config) {
     testFramework: 'jasmine',
     testRunner: 'jasmine',
     reporters: ['clear-text', 'event-recorder'],
-    maxConcurrentTestRunners: 1,
+    maxConcurrentTestRunners: 2,
     jasmineConfigFile: 'spec/support/jasmine.json',
-    fileLogLevel: 'debug'
+    fileLogLevel: 'debug',
+    plugins: ['@stryker-mutator/jasmine-runner']
   });
 };

e2e/test/jasmine-jasmine/verify/verify.ts

@@ -1,30 +1,28 @@
 import { promises as fs } from 'fs';
 
 import { expect } from 'chai';
-import { expectMetricsResult, produceMetrics } from '../../../helpers';
+import { expectMetrics } from '../../../helpers';
 
 describe('After running stryker with test runner jasmine, test framework jasmine', () => {
   it('should report 85% mutation score', async () => {
-    await expectMetricsResult({
-      metrics: produceMetrics({
-        killed: 12,
-        mutationScore: 85.71,
-        mutationScoreBasedOnCoveredCode: 92.31,
-        noCoverage: 1,
-        survived: 1,
-        totalCovered: 13,
-        totalDetected: 12,
-        totalMutants: 14,
-        totalUndetected: 2,
-        totalValid: 14
-      })
+    await expectMetrics({
+      killed: 12,
+      mutationScore: 85.71,
+      mutationScoreBasedOnCoveredCode: 100,
+      noCoverage: 2,
+      survived: 0,
+      totalCovered: 12,
+      totalDetected: 12,
+      totalMutants: 14,
+      totalUndetected: 2,
+      totalValid: 14
     });
   });
 
   it('should write to a log file', async () => {
     const strykerLog = await fs.readFile('./stryker.log', 'utf8');
     expect(strykerLog).matches(/INFO InputFileResolver Found 2 of 9 file\(s\) to be mutated/);
-    expect(strykerLog).matches(/Stryker Done in \d+/);
+    expect(strykerLog).matches(/Done in \d+ second/);
     // TODO, we now have an error because of a memory leak: https://github.com/jasmine/jasmine-npm/issues/134
     // expect(strykerLog).not.contains('ERROR');
   });

e2e/test/jest-react-ts/package.json

@@ -48,6 +48,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/typescript": "../../../packages/typescript",
     "@stryker-mutator/jest-runner": "../../../packages/jest-runner",
     "@stryker-mutator/util": "../../../packages/util"

e2e/test/jest-react/package.json

@@ -75,6 +75,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/javascript-mutator": "../../../packages/javascript-mutator",
     "@stryker-mutator/jest-runner": "../../../packages/jest-runner",
     "@stryker-mutator/util": "../../../packages/util"

e2e/test/karma-mocha/stryker.conf.js

@@ -13,6 +13,7 @@ module.exports = function (config) {
     },
     timeoutMS: 60000,
     maxConcurrentTestRunners: 2,
-    coverageAnalysis: 'perTest'
+    coverageAnalysis: 'perTest',
+    plugins: ['@stryker-mutator/karma-runner']
   });
 };

e2e/test/karma-mocha/verify/verify.ts

@@ -7,10 +7,10 @@ describe('Verify stryker has ran correctly', () => {
       metrics: produceMetrics({
         killed: 16,
         mutationScore: 64,
-        mutationScoreBasedOnCoveredCode: 84.21,
-        noCoverage: 6,
-        survived: 3,
-        totalCovered: 19,
+        mutationScoreBasedOnCoveredCode: 94.12,
+        noCoverage: 8,
+        survived: 1,
+        totalCovered: 17,
         totalDetected: 16,
         totalMutants: 25,
         totalUndetected: 9,

e2e/test/karma-webpack-with-ts/stryker.conf.json

@@ -15,5 +15,6 @@
     "configFile": "karma.conf.js",
     "projectType": "custom"
   },
-  "tsconfigFile": "tsconfig.json"
+  "tsconfigFile": "tsconfig.json",
+  "logLevel": "trace"
 }

e2e/test/polymer-project/package.json

@@ -28,6 +28,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/wct-runner": "../../../packages/wct-runner",
     "@stryker-mutator/javascript-mutator": "../../../packages/javascript-mutator",
     "@stryker-mutator/util": "../../../packages/util"

e2e/test/vue-cli-javascript-jest/package.json

@@ -25,6 +25,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/jest-runner": "../../../packages/jest-runner",
     "@stryker-mutator/vue-mutator": "../../../packages/vue-mutator",
     "@stryker-mutator/javascript-mutator": "../../../packages/javascript-mutator",

e2e/test/vue-cli-typescript-mocha/package.json

@@ -39,6 +39,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/mocha-runner": "../../../packages/mocha-runner",
     "@stryker-mutator/webpack-transpiler": "../../../packages/webpack-transpiler",
     "@stryker-mutator/vue-mutator": "../../../packages/vue-mutator",

e2e/test/vue-javascript/package.json

@@ -70,6 +70,7 @@
   "localDependencies": {
     "@stryker-mutator/api": "../../../packages/api",
     "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/instrumenter": "../../../packages/instrumenter",
     "@stryker-mutator/javascript-mutator": "../../../packages/javascript-mutator",
     "@stryker-mutator/karma-runner": "../../../packages/karma-runner",
     "@stryker-mutator/vue-mutator": "../../../packages/vue-mutator",

package.json

@@ -53,7 +53,7 @@
     "prebuild": "npm run generate",
     "build": "tsc -b && lerna run build",
     "test": "npm run mocha",
-    "mocha": "lerna run test --stream --concurrency 4",
+    "mocha": "lerna run test --stream --concurrency 4 --ignore @stryker-mutator/javascript-mutator --ignore @stryker-mutator/typescript",
     "e2e": "cd e2e && npm ci && npm t && cd ..",
     "perf": "cd perf && npm ci && npm t && cd ..",
     "start": "tsc -b -w",

packages/api/src/core/Mutant.ts

@@ -1,10 +1,12 @@
-import { Range } from '../../core';
+import Range from './Range';
+import Location from './Location';
 
 interface Mutant {
   id: number;
   mutatorName: string;
   fileName: string;
   range: Range;
+  location: Location;
   replacement: string;
 }
 

packages/api/src/report/MatchedMutant.ts

@@ -14,7 +14,7 @@ interface MatchedMutant {
   /**
    * If not all tests will run for this mutant, this array will contain the ids of the tests that will run.
    */
-  readonly scopedTestIds: number[];
+  readonly testFilter: string[] | undefined;
   /**
    * The time spent on the tests that will run in initial test run
    */

packages/api/src/test_runner2/DryRunResult.ts

@@ -1,5 +1,5 @@
 import { MutantCoverage } from './MutantCoverage';
-import { RunStatus } from './RunStatus';
+import { DryRunStatus } from './DryRunStatus';
 import { TestResult } from './TestResult';
 
 export type DryRunResult = CompleteDryRunResult | TimeoutDryRunResult | ErrorDryRunResult;
@@ -15,20 +15,20 @@ export interface CompleteDryRunResult {
   /**
    * The status of the run
    */
-  status: RunStatus.Complete;
+  status: DryRunStatus.Complete;
 }
 export interface TimeoutDryRunResult {
   /**
    * The status of the run
    */
-  status: RunStatus.Timeout;
+  status: DryRunStatus.Timeout;
 }
 
 export interface ErrorDryRunResult {
   /**
    * The status of the run
    */
-  status: RunStatus.Error;
+  status: DryRunStatus.Error;
 
   /**
    * If `state` is `error`, this collection should contain the error messages

packages/api/src/test_runner2/RunStatus.ts → packages/api/src/test_runner2/DryRunStatus.ts

@@ -1,4 +1,4 @@
-export enum RunStatus {
+export enum DryRunStatus {
   /**
    * Indicates that a test run is completed with failed or succeeded tests
    */

packages/api/src/test_runner2/runResultHelpers.ts

@@ -1,13 +1,13 @@
 import { TestStatus } from './TestStatus';
 import { DryRunResult } from './DryRunResult';
 import { MutantRunResult } from './MutantRunResult';
-import { RunStatus } from './RunStatus';
+import { DryRunStatus } from './DryRunStatus';
 import { MutantRunStatus } from './MutantRunResult';
 import { FailedTestResult } from './TestResult';
 
 export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult {
   switch (dryRunResult.status) {
-    case RunStatus.Complete: {
+    case DryRunStatus.Complete: {
       const killedBy = dryRunResult.tests.find<FailedTestResult>((test): test is FailedTestResult => test.status === TestStatus.Failed);
       if (killedBy) {
         return {
@@ -21,12 +21,12 @@ export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult {
         };
       }
     }
-    case RunStatus.Error:
+    case DryRunStatus.Error:
       return {
         status: MutantRunStatus.Error,
         errorMessage: dryRunResult.errorMessage,
       };
-    case RunStatus.Timeout:
+    case DryRunStatus.Timeout:
       return {
         status: MutantRunStatus.Timeout,
       };

packages/api/test/unit/test_runner2/runResultHelpers.spec.ts

@@ -1,25 +1,25 @@
 import { expect } from 'chai';
 
-import { toMutantRunResult, RunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner2';
+import { toMutantRunResult, DryRunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner2';
 import TestStatus from '../../../src/test_runner/TestStatus';
 
 describe('runResultHelpers', () => {
   describe(toMutantRunResult.name, () => {
     it('should convert "timeout" to "timeout"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Timeout };
-      expect(toMutantRunResult({ status: RunStatus.Timeout })).deep.eq(expected);
+      expect(toMutantRunResult({ status: DryRunStatus.Timeout })).deep.eq(expected);
     });
 
     it('should convert "error" to "error"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Error, errorMessage: 'some error' };
-      expect(toMutantRunResult({ status: RunStatus.Error, errorMessage: 'some error' })).deep.eq(expected);
+      expect(toMutantRunResult({ status: DryRunStatus.Error, errorMessage: 'some error' })).deep.eq(expected);
     });
 
     it('should report a failed test as "killed"', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Killed, failureMessage: 'expected foo to be bar', killedBy: '42' };
       expect(
         toMutantRunResult({
-          status: RunStatus.Complete,
+          status: DryRunStatus.Complete,
           tests: [
             { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
             { status: TestStatus.Failed, id: '42', name: 'error', timeSpentMs: 42, failureMessage: 'expected foo to be bar' },
@@ -33,7 +33,7 @@ describe('runResultHelpers', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Survived };
       expect(
         toMutantRunResult({
-          status: RunStatus.Complete,
+          status: DryRunStatus.Complete,
           tests: [
             { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
             { status: TestStatus.Success, id: '42', name: 'error', timeSpentMs: 42 },
@@ -47,7 +47,7 @@ describe('runResultHelpers', () => {
       const expected: MutantRunResult = { status: MutantRunStatus.Survived };
       expect(
         toMutantRunResult({
-          status: RunStatus.Complete,
+          status: DryRunStatus.Complete,
           tests: [],
         })
       ).deep.eq(expected);

packages/api/test_runner2.ts

@@ -5,5 +5,5 @@ export * from './src/test_runner2/DryRunResult';
 export * from './src/test_runner2/RunOptions';
 export * from './src/test_runner2/MutantCoverage';
 export * from './src/test_runner2/MutantRunResult';
-export * from './src/test_runner2/RunStatus';
+export * from './src/test_runner2/DryRunStatus';
 export * from './src/test_runner2/runResultHelpers';