test: debugging qol improvements

packages/amplify-e2e-core/src/asciinema-recorder.ts

@@ -53,6 +53,9 @@ export class Recorder {
     if (this.exitCode !== undefined) {
       throw new Error('Already executed. Please start a new instance');
     }
+    if (process.env.VERBOSE_LOGGING_DO_NOT_USE_IN_CI_OR_YOU_WILL_BE_FIRED) {
+      console.log(`Executing command [${this.cmd} ${this.args.join(' ')}]`);
+    }
     this.childProcess = pty.spawn(this.cmd, this.args, {
       name: 'xterm-color',
       cols: this.cols,

packages/amplify-e2e-core/src/index.ts

@@ -12,6 +12,7 @@ import { v4 as uuid } from 'uuid';
 import { pathManager } from 'amplify-cli-core';
 import { gt } from 'semver';
 import { sleep } from '.';
+import { isCI } from './utils';
 
 export * from './diagnose';
 export * from './configure';
@@ -120,8 +121,10 @@ export async function createNewProjectDir(
 
   fs.ensureDirSync(projectDir);
 
-  const initialDelay = Math.floor(Math.random() * 180 * 1000);
-  await sleep(initialDelay);
+  if (isCI()) {
+    const initialDelay = Math.floor(Math.random() * 180 * 1000);
+    await sleep(initialDelay);
+  }
 
   console.log(projectDir);
   return projectDir;

packages/amplify-e2e-core/src/utils/nexpect.ts

@@ -380,22 +380,15 @@ function chain(context: Context): ExecutionContext {
     let stdout: string[] = [];
     let noOutputTimer;
 
-    let logDumpFile: fs.WriteStream;
+    let stdoutPipe: NodeJS.WriteStream;
 
     if (process.env.VERBOSE_LOGGING_DO_NOT_USE_IN_CI_OR_YOU_WILL_BE_FIRED) {
-      const logdir = join(os.tmpdir(), 'amplify_e2e_logs');
-      fs.ensureDirSync(logdir);
-      const filename = join(logdir, `amplify_e2e_log_${generateRandomShortId()}`);
-      logDumpFile = fs.createWriteStream(filename);
-      console.log(`CLI test logs at [${filename}]`);
+      stdoutPipe = process.stdout;
     }
 
     const exitHandler = (code: number, signal: any) => {
       noOutputTimer.clear();
       context.process.removeOnExitHandlers(exitHandler);
-      if (logDumpFile) {
-        logDumpFile.close();
-      }
       if (code !== 0) {
         if (code === EXIT_CODE_TIMEOUT) {
           const recordings = context.process?.getRecordingFrames() || [];
@@ -547,8 +540,6 @@ function chain(context: Context): ExecutionContext {
       return undefined;
     }
 
-    const spinnerRegex = new RegExp(/.*(⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏).*/);
-
     //
     // **onLine**
     //
@@ -560,11 +551,11 @@ function chain(context: Context): ExecutionContext {
     // 3. Splitting `data` into multiple lines.
     //
     function onLine(data: string | Buffer) {
+      if (stdoutPipe) {
+        stdoutPipe.write(data);
+      }
       noOutputTimer.reschedule(context.noOutputTimeout);
       data = data.toString();
-      if (logDumpFile && spinnerRegex.test(data) === false && strip(data).trim().length > 0) {
-        logDumpFile.write(`${data}${EOL}`);
-      }
 
       if (context.stripColors) {
         data = strip(data);

packages/amplify-e2e-tests/Readme.md

@@ -26,37 +26,40 @@ npm run e2e src/__tests__/init.test.ts
 
 E2E tests internally use a forked version of [nexpect](https://www.npmjs.com/package/nexpect) to run the CLI. There are helper methods that helps you to set up and delete project. The recommended pattern is to create a helper method that creates a resources as a helper method so these method could be used in other tests. For instance, `initJSProjectWithProfile` is a helper method that is used in `init` tests and also used in all the other tests to initalize a new Javascript project. The tests should have all the assertions to make sure the resource created by the helper method is setup correctly. We recommend using `aws-sdk` to make assert the resource configuration.
 
-To configure the amount of time nexpect will wait for CLI responses, you can set the `AMPLIFY_TEST_TIMEOUT_SEC` environment variable. It is helpful to set this to a low value (10 seconds or so) when writing new tests so that you don't spend unnecessary time waiting for nexpect to error out on a misconfigured wait() block
+If you want to log the test results for debugging, set the environment variable `VERBOSE_LOGGING_DO_NOT_USE_IN_CI_OR_YOU_WILL_BE_FIRED` to `true` and CLI output will be piped to stdout of the test harness. This will allow you to see CLI output as if you had run the test commands directly.
 
-If you want to log the test results for debugging, set the environment variable `VERBOSE_LOGGING_DO_NOT_USE_IN_CI_OR_YOU_WILL_BE_FIRED` to `true` and output logs will be written to temp files. The temp file paths will be printed as the tests run and you can `cat` or `tail` the logs to see the CLI output
+> Note: it is not recommended to set this option if you are running multiple tests at once. This would cause the output from multiple tests to be interleaved together.
 
 ```sh
-env VERBOSE_LOGGING_DO_NOT_USE_IN_CI_OR_YOU_WILL_BE_FIRED=true yarn e2e
+env VERBOSE_LOGGING_DO_NOT_USE_IN_CI_OR_YOU_WILL_BE_FIRED=true yarn e2e <path/to/test/file> -t 'name of individual test'
 ```
 
 ```typescript
 import { amplifyPush, deleteProject, initJSProjectWithProfile } from '../init';
 import { createNewProjectDir, deleteProjectDir, getProjectMeta } from '../utils';
 
-describe('amplify your test', () => {
+describe('name of component or feature', () => {
   let projRoot: string;
   beforeEach(() => {
     projRoot = createNewProjectDir(); // create a new project for each test
-    jest.setTimeout(1000 * 60 * 60); // 1 hour timeout as pushing might be slow
+    // initialize an Amplify Project in the directory
+    // If you are testing a specific init behavior, this step might be different, but most e2e tests initialize in this way.
+    await initJSProjectWithProfile(projRoot, { name: '<project-name>' });
   });
 
   afterEach(async () => {
     await deleteProject(projRoot); // delete the project from the cloud
     deleteProjectDir(projRoot); // delete the project directory
   });
 
-  it('<your test>', async () => {
-    await initJSProjectWithProfile(projRoot, { name: '<project-name>' });
+  it('specific behavior description', async () => {
     // add resources that you want to test
+    await addFunction(projRoot, { functionTemplate: 'Hello World' }, 'nodejs');
     await amplifyPush(projRoot); // Push it to the cloud
-    const { output } = getProjectMeta(projRoot).api.simplemodel;
 
-    // TODO - assertion to make sure the resources are pushed. Use matcher
+    // add normal jest assertions about project state
+    const { output } = getProjectMeta(projRoot).api.simplemodel;
+    expect(output).toBeDefined();
   });
 });
 ```
@@ -72,4 +75,4 @@ There are two scenarios when this approach can cause trouble:
 1. Locally running two or more test suites in parallel with this test included, the other tests might fail because test project can not be init'ed when the above mentioned config and credential files are missing.
 2. In the middle of execution, the test is interrupted by Ctrl+C, then the hidden config and credential files are not renamed back.
 
-So, You should NOT run multiple tests in parallel locally with the `init-special-case` test included. And, if you use Ctrl+C to interrupt the `init-special-case` test, you need to go to the `~/.aws/c` folder and rename the config and credential files to their original names.
+So, You should NOT run multiple tests in parallel locally with the `init-special-case` test included. And, if you use Ctrl+C to interrupt the `init-special-case` test, you need to go to the `~/.aws` folder and rename the config and credential files to their original names.

packages/amplify-e2e-tests/src/setup-tests.ts

@@ -1,3 +1,4 @@
+import { isCI } from '@aws-amplify/amplify-e2e-core';
 import { toBeIAMRoleWithArn, toHaveValidPolicyConditionMatchingIdpId, toBeAS3Bucket } from './aws-matchers';
 
 expect.extend({ toBeIAMRoleWithArn });
@@ -7,4 +8,6 @@ expect.extend({ toBeAS3Bucket });
 const JEST_TIMEOUT = 1000 * 60 * 60; // 1 hour
 
 jest.setTimeout(JEST_TIMEOUT);
-jest.retryTimes(1);
+if (isCI()) {
+  jest.retryTimes(1);
+}