fix: issue with compilation failures in component testing (#21599)

* fix: issue with compilation failures in component testing

* add tests

* fix tests

* Refactor tests

* Fix tests

* Refactor tests

* Fix tests

* Fix paths for dependencies in system tests

* Fix tests

* Fix tests

* Fix tests

* Fix tests

* test fix for initial esbuild failures

* Fix tests

* Add back ESLintPlugin

* Add comments around our special esbuild handling logic in vite

* Code cleanup and additional test scenario

* Add config syntax error tests

* Improve tests

* Update comment and remove unused variable

* Remove unneeded hook in webpack dev server

* Disable dev server overlay

* Revert "Remove unneeded hook in webpack dev server"

This reverts commit 98b2f26.

* PR comments

* Accidental removal

* Fix dedent

* PR comments

npm/vite-dev-server/cypress/e2e/react.cy.ts

@@ -1,6 +1,7 @@
 /// <reference types="cypress" />
 /// <reference path="../support/e2e.ts" />
 import type { fixtureDirs } from '@tooling/system-tests'
+import dedent from 'dedent'
 
 type ProjectDirs = typeof fixtureDirs
 
@@ -24,17 +25,19 @@ for (const project of VITE_REACT) {
     it('should mount a passing test', () => {
       cy.visitApp()
       cy.contains('App.cy.jsx').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
     })
 
     it('MissingReact: should fail, rerun, succeed', () => {
-      cy.once('uncaught:exception', () => {
+      cy.on('uncaught:exception', () => {
         // Ignore the uncaught exception in the AUT
         return false
       })
 
       cy.visitApp()
       cy.contains('MissingReact.cy.jsx').click()
+      cy.waitForSpecToFinish()
       cy.get('.failed > .num').should('contain', 1)
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject(`src/MissingReact.jsx`,
@@ -46,8 +49,14 @@ for (const project of VITE_REACT) {
     })
 
     it('MissingReactInSpec: should fail, rerun, succeed', () => {
+      cy.on('uncaught:exception', () => {
+        // Ignore the uncaught exception in the AUT
+        return false
+      })
+
       cy.visitApp()
       cy.contains('MissingReactInSpec.cy.jsx').click()
+      cy.waitForSpecToFinish()
       cy.get('.failed > .num').should('contain', 1)
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject(`src/MissingReactInSpec.cy.jsx`,
@@ -56,5 +65,55 @@ for (const project of VITE_REACT) {
 
       cy.get('.passed > .num').should('contain', 1)
     })
+
+    it('AppCompilationError: should fail with uncaught exception error', () => {
+      cy.on('uncaught:exception', () => {
+        // Ignore the uncaught exception in the AUT
+        return false
+      })
+
+      cy.visitApp()
+      cy.contains('AppCompilationError.cy.jsx').click()
+      cy.waitForSpecToFinish()
+      cy.get('.failed > .num').should('contain', 1)
+      cy.contains('An uncaught error was detected outside of a test')
+      cy.contains('The following error originated from your test code, not from Cypress.')
+
+      // Correct the problem
+      cy.withCtx(async (ctx) => {
+        await ctx.actions.file.writeFileInProject(
+          `src/AppCompilationError.cy.jsx`,
+          await ctx.file.readFileInProject('src/App.cy.jsx'),
+        )
+      })
+
+      cy.waitForSpecToFinish()
+      cy.get('.passed > .num').should('contain', 1)
+
+      const appCompilationErrorSpec = dedent`
+        import React from 'react'
+        import { mount } from 'cypress/react'
+        import { App } from './App'
+
+        it('renders hello world', () => {
+          mount(<App />)
+          cy.get('h1').contains('Hello World')
+        }
+        })
+      `
+
+      // Cause the problem again
+      cy.withCtx(async (ctx, o) => {
+        await ctx.actions.file.writeFileInProject(
+          `src/AppCompilationError.cy.jsx`,
+          o.appCompilationErrorSpec,
+        )
+      }, { appCompilationErrorSpec })
+
+      cy.waitForSpecToFinish()
+      cy.get('.failed > .num').should('contain', 1)
+      cy.contains('An uncaught error was detected outside of a test')
+      cy.contains('The following error originated from your test code, not from Cypress.')
+    })
   })
 }

npm/vite-dev-server/cypress/e2e/vite-dev-server.cy.ts

@@ -9,6 +9,7 @@ describe('Config options', () => {
 
     cy.visitApp()
     cy.contains('App.cy.jsx').click()
+    cy.waitForSpecToFinish()
     cy.get('.passed > .num').should('contain', 1)
   })
 

npm/vite-dev-server/cypress/support/commands.ts

@@ -1,37 +1,34 @@
 /// <reference types="cypress" />
-// ***********************************************
-// This example commands.ts shows you how to
-// create various custom commands and overwrite
-// existing commands.
-//
-// For more comprehensive examples of custom
-// commands please read more here:
-// https://on.cypress.io/custom-commands
-// ***********************************************
-//
-//
-// -- This is a parent command --
-// Cypress.Commands.add('login', (email, password) => { ... })
-//
-//
-// -- This is a child command --
-// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
-//
-//
-// -- This is a dual command --
-// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
-//
-//
-// -- This will overwrite an existing command --
-// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })
-//
-// declare global {
-//   namespace Cypress {
-//     interface Chainable {
-//       login(email: string, password: string): Chainable<void>
-//       drag(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
-//       dismiss(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
-//       visit(originalFn: CommandOriginalFn, url: string, options: Partial<VisitOptions>): Chainable<Element>
-//     }
-//   }
-// }
+
+declare global {
+  namespace Cypress {
+    interface Chainable {
+      /**
+       * Adapter to wait for a spec to finish in a standard way. It
+       *
+       * 1. Waits for the stats to reset which signifies that the test page has loaded
+       * 2. Waits for 'Your tests are loading...' to not be present so that we know the tests themselves have loaded
+       * 3. Waits (with a timeout of 30s) for the Rerun all tests button to be present. This ensures all tests have completed
+       *
+       */
+      waitForSpecToFinish()
+    }
+  }
+}
+
+// Here we export the function with no intention to import it
+// This only tells the typescript type checker that this definitely is a module
+// This way, we are allowed to use the global namespace declaration
+export const waitForSpecToFinish = () => {
+  // First ensure the test is loaded
+  cy.get('.passed > .num').should('contain', '--')
+  cy.get('.failed > .num').should('contain', '--')
+
+  // Then ensure the tests are running
+  cy.contains('Your tests are loading...').should('not.exist')
+
+  // Then ensure the tests have finished
+  cy.get('[aria-label="Rerun all tests"]', { timeout: 30000 })
+}
+
+Cypress.Commands.add('waitForSpecToFinish', waitForSpecToFinish)

npm/vite-dev-server/cypress/support/e2e.ts

@@ -15,6 +15,4 @@
 
 // Import commands.js using ES2015 syntax:
 import '@packages/frontend-shared/cypress/e2e/support/e2eSupport'
-
-// Alternatively you can use CommonJS syntax:
-// require('./commands')
+import './commands'

npm/vite-dev-server/package.json

@@ -21,6 +21,7 @@
     "pathe": "0.2.0"
   },
   "devDependencies": {
+    "dedent": "^0.7.0",
     "vite": "2.9.0-beta.3",
     "vite-plugin-inspect": "0.4.3"
   },

npm/vite-dev-server/src/resolveConfig.ts

@@ -46,6 +46,24 @@ export const createViteDevServerConfig = async (config: ViteDevServerConfig, vit
     base: `${cypressConfig.devServerPublicPathRoute}/`,
     configFile,
     optimizeDeps: {
+      esbuildOptions: {
+        incremental: true,
+        plugins: [
+          {
+            name: 'cypress-esbuild-plugin',
+            setup (build) {
+              build.onEnd(function (result) {
+                // We don't want to completely fail the build here on errors so we treat the errors as warnings
+                // which will handle things more gracefully. Vite will 500 on files that have errors when they
+                // are requested later and Cypress will display an error message.
+                // See: https://github.com/cypress-io/cypress/pull/21599
+                result.warnings = [...result.warnings, ...result.errors]
+                result.errors = []
+              })
+            },
+          },
+        ],
+      },
       entries: [
         ...specs.map((s) => relative(root, s.relative)),
         ...(cypressConfig.supportFile ? [resolve(root, cypressConfig.supportFile)] : []),

npm/webpack-dev-server/__snapshots__/makeWebpackConfig.spec.ts.js

@@ -1,8 +1,36 @@
-exports['makeWebpackConfig ignores userland webpack `output.publicPath` 1'] = {
+exports['makeWebpackConfig ignores userland webpack `output.publicPath` and `devServer.overlay` with webpack-dev-server v4 1'] = {
   "output": {
     "publicPath": "/test-public-path/",
     "filename": "[name].js"
   },
+  "devServer": {
+    "magicHtml": true,
+    "client": {
+      "progress": false,
+      "overlay": false
+    }
+  },
+  "mode": "development",
+  "optimization": {
+    "splitChunks": {
+      "chunks": "all"
+    }
+  },
+  "plugins": [
+    "HtmlWebpackPlugin",
+    "CypressCTWebpackPlugin"
+  ]
+}
+
+exports['makeWebpackConfig ignores userland webpack `output.publicPath` and `devServer.overlay` with webpack-dev-server v3 1'] = {
+  "output": {
+    "publicPath": "/test-public-path/",
+    "filename": "[name].js"
+  },
+  "devServer": {
+    "progress": true,
+    "overlay": false
+  },
   "mode": "development",
   "optimization": {
     "splitChunks": {

npm/webpack-dev-server/cypress/e2e/create-react-app.cy.ts

@@ -22,13 +22,15 @@ for (const project of WEBPACK_REACT) {
     it('should mount a passing test', () => {
       cy.visitApp()
       cy.contains('App.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
     })
 
     it('should live-reload on src changes', () => {
       cy.visitApp()
 
       cy.contains('App.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
 
       cy.withCtx(async (ctx) => {
@@ -61,6 +63,7 @@ for (const project of WEBPACK_REACT) {
       })
 
       cy.contains('New.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
     })
   })

npm/webpack-dev-server/cypress/e2e/next.cy.ts

@@ -22,13 +22,15 @@ for (const project of WEBPACK_REACT) {
     it('should mount a passing test', () => {
       cy.visitApp()
       cy.contains('index.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
     })
 
     it('should live-reload on src changes', () => {
       cy.visitApp()
 
       cy.contains('index.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
 
       cy.withCtx(async (ctx) => {
@@ -68,6 +70,7 @@ for (const project of WEBPACK_REACT) {
       })
 
       cy.contains('New.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
     })
   })

npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts

@@ -24,6 +24,7 @@ for (const project of PROJECTS) {
     it('should mount a passing test and live-reload', () => {
       cy.visitApp()
       cy.contains('Tutorial.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
 
       cy.withCtx(async (ctx) => {
@@ -63,6 +64,7 @@ for (const project of PROJECTS) {
       })
 
       cy.contains('New.cy.js').click()
+      cy.waitForSpecToFinish()
       cy.get('.passed > .num').should('contain', 1)
     })
   })