diff --git a/packages/@aws-cdk/integ-runner/README.md b/packages/@aws-cdk/integ-runner/README.md index 6fb4a983d339a..bfe58e2ff4926 100644 --- a/packages/@aws-cdk/integ-runner/README.md +++ b/packages/@aws-cdk/integ-runner/README.md @@ -85,7 +85,7 @@ If you are providing a list of tests to execute, either as CLI arguments or from For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js` ```bash -yarn integ integ.policy.js +integ-runner integ.policy.js ``` ### Common Workflow @@ -121,7 +121,7 @@ Snapshot Results: Tests: 1 failed, 9 total Error: Some snapshot tests failed! -To re-run failed tests run: yarn integ-runner --update-on-failed +To re-run failed tests run: integ-runner --update-on-failed at main (packages/@aws-cdk/integ-runner/lib/cli.js:90:15) error Command failed with exit code 1. ``` @@ -197,7 +197,7 @@ If you are adding a new test which creates a new snapshot then you should run th For example, if you are working on a new test `integ.new-test.js` then you would run: ```bash -yarn integ --update-on-failed --disable-update-workflow integ.new-test.js +integ-runner --update-on-failed --disable-update-workflow integ.new-test.js ``` This is because for a new test we do not need to test the update workflow (there is nothing to update). @@ -210,3 +210,25 @@ See [@aws-cdk/cloud-assembly-schema/lib/integ-tests/schema.ts](../cloud-assembly See the `@aws-cdk/integ-tests` module for information on how to define integration tests for the runner to exercise. + +### Config file + +All options can be configured via the `integ.config.json` configuration file in the current working directory. + +```json +{ + "maxWorkers": 10, + "parallelRegions": [ + "eu-west-1", + "ap-southeast-2" + ] +} +``` + +Available options can be listed by running the following command: + +```sh +integ-runner --help +``` + +To use a different config file, provide the `--config` command-line option. diff --git a/packages/@aws-cdk/integ-runner/lib/cli.ts b/packages/@aws-cdk/integ-runner/lib/cli.ts index c3a2256c0813b..8356ea91e4a43 100644 --- a/packages/@aws-cdk/integ-runner/lib/cli.ts +++ b/packages/@aws-cdk/integ-runner/lib/cli.ts @@ -1,10 +1,10 @@ // Exercise all integ stacks and if they deploy, update the expected synth files -import { promises as fs } from 'fs'; +import * as fs from 'fs'; import * as path from 'path'; import * as chalk from 'chalk'; import * as workerpool from 'workerpool'; import * as logger from './logger'; -import { IntegrationTests, IntegTestInfo, IntegTest } from './runner/integration-tests'; +import { IntegrationTests, IntegTestInfo } from './runner/integration-tests'; import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers'; // https://github.com/yargs/yargs/issues/1929 @@ -12,10 +12,15 @@ import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWor // eslint-disable-next-line @typescript-eslint/no-require-imports const yargs = require('yargs'); - -export async function main(args: string[]) { +export function parseCliArgs(args: string[] = []) { const argv = yargs .usage('Usage: integ-runner [TEST...]') + .option('config', { + config: true, + configParser: IntegrationTests.configFromFile, + default: 'integ.config.json', + desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.', + }) .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' }) .option('clean', { type: 'boolean', default: true, desc: 'Skips stack clean up after test is completed (use --no-clean to negate)' }) .option('verbose', { type: 'boolean', default: false, alias: 'v', count: true, desc: 'Verbose logs and metrics on integration tests durations (specify multiple times to increase verbosity)' }) @@ -35,61 +40,87 @@ export async function main(args: string[]) { .strict() .parse(args); - const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), { - maxWorkers: argv['max-workers'], - }); - - // list of integration tests that will be executed - const testRegex = arrayFromYargs(argv['test-regex']); - const testsToRun: IntegTestWorkerConfig[] = []; - const destructiveChanges: DestructiveChange[] = []; - const testsFromArgs: IntegTest[] = []; + const tests: string[] = argv._; const parallelRegions = arrayFromYargs(argv['parallel-regions']); const testRegions: string[] = parallelRegions ?? ['us-east-1', 'us-east-2', 'us-west-2']; const profiles = arrayFromYargs(argv.profiles); - const runUpdateOnFailed = argv['update-on-failed'] ?? false; const fromFile: string | undefined = argv['from-file']; - const exclude: boolean = argv.exclude; - const app: string | undefined = argv.app; + const maxWorkers: number = argv['max-workers']; + const verbosity: number = argv.verbose; + const verbose: boolean = verbosity >= 1; - let failedSnapshots: IntegTestWorkerConfig[] = []; - if (argv['max-workers'] < testRegions.length * (profiles ?? [1]).length) { - logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', argv.profiles * argv['parallel-regions'], argv['max-workers']); + const numTests = testRegions.length * (profiles ?? [1]).length; + if (maxWorkers < numTests) { + logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', numTests, maxWorkers); } - let testsSucceeded = false; - try { - if (argv.list) { - const tests = await new IntegrationTests(argv.directory).fromCliArgs({ testRegex, app }); - process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n'); - return; - } + if (tests.length > 0 && fromFile) { + throw new Error('A list of tests cannot be provided if "--from-file" is provided'); + } + const requestedTests = fromFile + ? (fs.readFileSync(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x) + : (tests.length > 0 ? tests : undefined); // 'undefined' means no request + + return { + tests: requestedTests, + app: argv.app as (string | undefined), + testRegex: arrayFromYargs(argv['test-regex']), + testRegions, + profiles, + runUpdateOnFailed: (argv['update-on-failed'] ?? false) as boolean, + fromFile, + exclude: argv.exclude as boolean, + maxWorkers, + list: argv.list as boolean, + directory: argv.directory as string, + inspectFailures: argv['inspect-failures'] as boolean, + verbosity, + verbose, + clean: argv.clean as boolean, + force: argv.force as boolean, + dryRun: argv['dry-run'] as boolean, + disableUpdateWorkflow: argv['disable-update-workflow'] as boolean, + }; +} - if (argv._.length > 0 && fromFile) { - throw new Error('A list of tests cannot be provided if "--from-file" is provided'); - } - const requestedTests = fromFile - ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x) - : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request - testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs({ - app, - testRegex, - tests: requestedTests, - exclude, - }))); +export async function main(args: string[]) { + const options = parseCliArgs(args); + + const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({ + app: options.app, + testRegex: options.testRegex, + tests: options.tests, + exclude: options.exclude, + }); + + // List only prints the discoverd tests + if (options.list) { + process.stdout.write(testsFromArgs.map(t => t.discoveryRelativeFileName).join('\n') + '\n'); + return; + } + + const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), { + maxWorkers: options.maxWorkers, + }); + const testsToRun: IntegTestWorkerConfig[] = []; + const destructiveChanges: DestructiveChange[] = []; + let failedSnapshots: IntegTestWorkerConfig[] = []; + let testsSucceeded = false; + + try { // always run snapshot tests, but if '--force' is passed then // run integration tests on all failed tests, not just those that // failed snapshot tests failedSnapshots = await runSnapshotTests(pool, testsFromArgs, { - retain: argv['inspect-failures'], - verbose: Boolean(argv.verbose), + retain: options.inspectFailures, + verbose: options.verbose, }); for (const failure of failedSnapshots) { destructiveChanges.push(...failure.destructiveChanges ?? []); } - if (!argv.force) { + if (!options.force) { testsToRun.push(...failedSnapshots); } else { // if any of the test failed snapshot tests, keep those results @@ -98,25 +129,25 @@ export async function main(args: string[]) { } // run integration tests if `--update-on-failed` OR `--force` is used - if (runUpdateOnFailed || argv.force) { + if (options.runUpdateOnFailed || options.force) { const { success, metrics } = await runIntegrationTests({ pool, tests: testsToRun, - regions: testRegions, - profiles, - clean: argv.clean, - dryRun: argv['dry-run'], - verbosity: argv.verbose, - updateWorkflow: !argv['disable-update-workflow'], + regions: options.testRegions, + profiles: options.profiles, + clean: options.clean, + dryRun: options.dryRun, + verbosity: options.verbosity, + updateWorkflow: !options.disableUpdateWorkflow, }); testsSucceeded = success; - if (argv.clean === false) { + if (options.clean === false) { logger.warning('Not cleaning up stacks since "--no-clean" was used'); } - if (Boolean(argv.verbose)) { + if (Boolean(options.verbose)) { printMetrics(metrics); } @@ -134,8 +165,8 @@ export async function main(args: string[]) { } if (failedSnapshots.length > 0) { let message = ''; - if (!runUpdateOnFailed) { - message = 'To re-run failed tests run: yarn integ-runner --update-on-failed'; + if (!options.runUpdateOnFailed) { + message = 'To re-run failed tests run: integ-runner --update-on-failed'; } if (!testsSucceeded) { throw new Error(`Some tests failed!\n${message}`); diff --git a/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts b/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts index d7559ea911c10..fa4d64a4d3876 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts @@ -175,32 +175,26 @@ export interface IntegrationTestsDiscoveryOptions { } -/** - * The list of tests to run can be provided in a file - * instead of as command line arguments. - */ -export interface IntegrationTestFileConfig extends IntegrationTestsDiscoveryOptions { - /** - * List of tests to include (or exclude if `exclude=true`) - */ - readonly tests: string[]; -} - /** * Discover integration tests */ export class IntegrationTests { - constructor(private readonly directory: string) { - } - /** - * Takes a file name of a file that contains a list of test - * to either run or exclude and returns a list of Integration Tests to run + * Return configuration options from a file */ - public async fromFile(fileName: string): Promise { - const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' })); + public static configFromFile(fileName?: string): Record { + if (!fileName) { + return {}; + } - return this.discover(file); + try { + return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' })); + } catch { + return {}; + } + } + + constructor(private readonly directory: string) { } /** diff --git a/packages/@aws-cdk/integ-runner/lib/utils.ts b/packages/@aws-cdk/integ-runner/lib/utils.ts index c783e6ca2a491..d6b373e3f16f8 100644 --- a/packages/@aws-cdk/integ-runner/lib/utils.ts +++ b/packages/@aws-cdk/integ-runner/lib/utils.ts @@ -75,6 +75,7 @@ export class WorkList { public done() { this.remaining.clear(); + this.stopTimer(); } private stopTimer() { diff --git a/packages/@aws-cdk/integ-runner/test/cli.test.ts b/packages/@aws-cdk/integ-runner/test/cli.test.ts index cd17ce7205d63..87bfabd0e7e78 100644 --- a/packages/@aws-cdk/integ-runner/test/cli.test.ts +++ b/packages/@aws-cdk/integ-runner/test/cli.test.ts @@ -1,5 +1,15 @@ +import * as fs from 'fs'; +import * as os from 'os'; import * as path from 'path'; -import { main } from '../lib/cli'; +import { main, parseCliArgs } from '../lib/cli'; + +let stdoutMock: jest.SpyInstance; +beforeEach(() => { + stdoutMock = jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); +}); +afterEach(() => { + stdoutMock.mockRestore(); +}); describe('CLI', () => { const currentCwd = process.cwd(); @@ -10,14 +20,6 @@ describe('CLI', () => { process.chdir(currentCwd); }); - let stdoutMock: jest.SpyInstance; - beforeEach(() => { - stdoutMock = jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); - }); - afterEach(() => { - stdoutMock.mockRestore(); - }); - test('find by default pattern', async () => { await main(['--list', '--directory=test/test-data']); @@ -36,4 +38,67 @@ describe('CLI', () => { ].join('\n'), ]]); }); + + test('list only shows explicitly provided tests', async () => { + await main(['xxxxx.integ-test1.js', 'xxxxx.integ-test2.js', '--list', '--directory=test/test-data', '--test-regex="^xxxxx\..*\.js$"']); + + expect(stdoutMock.mock.calls).toEqual([[ + [ + 'xxxxx.integ-test1.js', + 'xxxxx.integ-test2.js', + '', + ].join('\n'), + ]]); + }); + + test('can run with no tests detected', async () => { + await main(['whatever.js', '--directory=test/test-data']); + + expect(stdoutMock.mock.calls).toEqual([]); + }); +}); + +describe('CLI config file', () => { + const configFile = 'integ.config.json'; + const withConfig = (settings: any, fileName = configFile) => { + fs.writeFileSync(fileName, JSON.stringify(settings, null, 2), { encoding: 'utf-8' }); + }; + + const currentCwd = process.cwd(); + beforeEach(() => { + process.chdir(os.tmpdir()); + }); + afterEach(() => { + process.chdir(currentCwd); + }); + + test('options are read from config file', async () => { + // WHEN + withConfig({ + list: true, + maxWorkers: 3, + parallelRegions: [ + 'eu-west-1', + 'ap-southeast-2', + ], + }); + const options = parseCliArgs(); + + // THEN + expect(options.list).toBe(true); + expect(options.maxWorkers).toBe(3); + expect(options.testRegions).toEqual([ + 'eu-west-1', + 'ap-southeast-2', + ]); + }); + + test('cli options take precedent', async () => { + // WHEN + withConfig({ maxWorkers: 3 }); + const options = parseCliArgs(['--max-workers', '20']); + + // THEN + expect(options.maxWorkers).toBe(20); + }); }); diff --git a/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts b/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts index 9b93709abdc75..010377436b8f0 100644 --- a/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts +++ b/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts @@ -1,6 +1,5 @@ -import { writeFileSync } from 'fs'; import * as mockfs from 'mock-fs'; -import { IntegrationTests, IntegrationTestsDiscoveryOptions } from '../../lib/runner/integration-tests'; +import { IntegrationTests } from '../../lib/runner/integration-tests'; describe('IntegrationTests', () => { const tests = new IntegrationTests('test'); @@ -90,81 +89,4 @@ describe('IntegrationTests', () => { expect(integTests[0].appCommand).toEqual('node --no-warnings {filePath}'); }); }); - - describe('from file', () => { - const configFile = 'integ.config.json'; - const writeConfig = (settings: IntegrationTestsDiscoveryOptions, fileName = configFile) => { - writeFileSync(fileName, JSON.stringify(settings, null, 2), { encoding: 'utf-8' }); - }; - - test('find all', async () => { - writeConfig({}); - const integTests = await tests.fromFile(configFile); - - expect(integTests.length).toEqual(3); - expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/)); - expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/)); - expect(integTests[2].fileName).toEqual(expect.stringMatching(/integ.integ-test3.js$/)); - }); - - - test('find named tests', async () => { - writeConfig({ tests: ['test-data/integ.integ-test1.js'] }); - const integTests = await tests.fromFile(configFile); - - expect(integTests.length).toEqual(1); - expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/)); - }); - - - test('test not found', async () => { - writeConfig({ tests: ['test-data/integ.integ-test16.js'] }); - const integTests = await tests.fromFile(configFile); - - expect(integTests.length).toEqual(0); - expect(stderrMock.mock.calls[0][0]).toContain( - 'No such integ test: test-data/integ.integ-test16.js', - ); - expect(stderrMock.mock.calls[1][0]).toContain( - 'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js', - ); - }); - - test('exclude tests', async () => { - writeConfig({ tests: ['test-data/integ.integ-test1.js'], exclude: true }); - const integTests = await tests.fromFile(configFile); - - const fileNames = integTests.map(test => test.fileName); - expect(integTests.length).toEqual(2); - expect(fileNames).not.toContain( - 'test/test-data/integ.integ-test1.js', - ); - }); - - test('match regex', async () => { - writeConfig({ testRegex: ['1\.js$', '2\.js'] }); - const integTests = await tests.fromFile(configFile); - - expect(integTests.length).toEqual(2); - expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/)); - expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/)); - }); - - test('match regex with path', async () => { - writeConfig({ testRegex: ['other-data/integ\..*\.js$'] }); - const otherTestDir = new IntegrationTests('.'); - const integTests = await otherTestDir.fromFile(configFile); - - expect(integTests.length).toEqual(1); - expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.other-test1.js$/)); - }); - - test('can set app command', async () => { - writeConfig({ app: 'node --no-warnings {filePath}' }); - const integTests = await tests.fromFile(configFile); - - expect(integTests.length).toEqual(3); - expect(integTests[0].appCommand).toEqual('node --no-warnings {filePath}'); - }); - }); });