tests: Add visual regression tests (#27619)

* tests: Add visual regression tests

* Add config

* Wait for images to load

* Do multiple window sizes

* Adjust settings for comparisons

* Fix wrapper

* Add 1024 window

* Save reports

* Update fixed image too large snapshot

* Disable video

* Update readme

* Update element id

* Force 1x pixel density

* Update readme

.circleci/config.yml

@@ -375,6 +375,17 @@ jobs:
           test_path: e2e-tests/gatsby-static-image
           test_command: yarn test
 
+  e2e_tests_visual-regression:
+    <<: *e2e-executor
+    steps:
+      - e2e-test:
+          test_path: e2e-tests/visual-regression
+          test_command: yarn test
+      - store_artifacts:
+          path: e2e-tests/visual-regression/__diff_output__
+      - store_test_results:
+          path: e2e-tests/visual-regression/cypress/results
+
   starters_validate:
     executor: node
     steps:
@@ -582,6 +593,8 @@ workflows:
           <<: *e2e-test-workflow
       - e2e_tests_gatsby-static-image:
           <<: *e2e-test-workflow
+      - e2e_tests_visual-regression:
+          <<: *e2e-test-workflow
       - e2e_tests_development_runtime:
           <<: *e2e-test-workflow
       - e2e_tests_production_runtime:

e2e-tests/visual-regression/.gitignore

@@ -0,0 +1,14 @@
+# Project dependencies
+.cache
+node_modules
+yarn-error.log
+
+# Build directory
+/public
+.DS_Store
+
+# Cypress output
+cypress/videos/
+cypress/screenshots/
+cypress/results/
+__diff_output__

e2e-tests/visual-regression/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

e2e-tests/visual-regression/README.md

@@ -0,0 +1,23 @@
+# Visual regression tests
+
+This test suite uses [cypress-image-snapshot](https://github.com/jaredpalmer/cypress-image-snapshot)
+to compare screenshots of pages or elements with a saved snapshot.
+
+To add a test, add a page to `src/pages`, then add a test to `cypress/integration`, or add to an existing spec.
+
+If tests fail, a comparison image will be written to `__diff_output__`. When running in CircleCI, this is uploaded to artifacts.
+
+## Considerations
+
+Remember that the test will run on Linux in CI, so avoid tests that might change between platforms.
+Using default fonts is an example. In general, if you're not testing the text itself then exclude it from your tests. Rather than comparing the full page, a good idea is to compare a wrapper element. There is a component provided for this purpose for images.
+
+Specifying large screen sizes can also cause problems when running locally on a small screen. The image tests use a maximum of 1024x768. The device pixel density is forced to 1, so running tests will look strange on Retina screens. This is to ensure screenshots match, whichever monitor or headless CI the tests rae running on.
+
+## Updating snapshots
+
+Run `yarn cy:update-snapshots` if you need to update them. Please note that unlike Jest, this doesn't delete outdated snapshots, so if you remove a test make sure to remove its snapshots too.
+
+## Credits
+
+Test images of Cornwall by [Benjamin Elliott](https://unsplash.com/photos/lH0_kBu5iyo) and [Red Zeppelin](https://unsplash.com/photos/uJMxXtH-Qso) via Unsplash.

e2e-tests/visual-regression/cypress.json

@@ -0,0 +1,11 @@
+{
+  "baseUrl": "http://localhost:9000",
+  "video": false,
+  "reporter": "junit",
+  "reporterOptions": {
+    "mochaFile": "cypress/results/junit-[hash].xml",
+    "overwrite": false,
+    "html": false,
+    "json": true
+  }
+}

e2e-tests/visual-regression/cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "[email protected]",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

e2e-tests/visual-regression/cypress/integration/image.js

@@ -0,0 +1,26 @@
+const testCases = [
+  ["fixed image", "/images/fixed"],
+  ["fixed image smaller than requested size", "/images/fixed-too-big"],
+  ["fluid image", "/images/fluid"],
+  ["constrained image", "/images/constrained"],
+]
+
+const sizes = [["iphone-6"], ["ipad-2"], [1027, 768]]
+
+describe(`GatsbyImage`, () => {
+  sizes.forEach(size => {
+    testCases.forEach(([title, path]) => {
+      describe(`${title}`, () => {
+        it(`renders correctly on ${size.join("x")}`, () => {
+          cy.viewport(...size)
+          cy.visit(path)
+          // Wait for main image to load
+          cy.get("[data-main-image]").should("have.css", "opacity", "1")
+          // Wait for blur-up
+          cy.wait(1000)
+          cy.get("#test-wrapper").matchImageSnapshot()
+        })
+      })
+    })
+  })
+})

e2e-tests/visual-regression/cypress/plugins/index.js

@@ -0,0 +1,24 @@
+// ***********************************************************
+// This example plugins/index.js can be used to load plugins
+//
+// You can change the location of this file or turn off loading
+// the plugins file with the 'pluginsFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/plugins-guide
+// ***********************************************************
+
+// This function is called when a project is opened or re-opened (e.g. due to
+// the project's config changing)
+const { addMatchImageSnapshotPlugin } = require("cypress-image-snapshot/plugin")
+
+module.exports = (on, config) => {
+  addMatchImageSnapshotPlugin(on, config)
+  on("before:browser:launch", (browser = {}, launchOptions) => {
+    if (browser.family === "chromium" || browser.family === "chrome") {
+      // Make retina screens run at 1x density so they match the versions in CI
+      launchOptions.push("--force-device-scale-factor=1")
+    }
+    return launchOptions
+  })
+}

e2e-tests/visual-regression/cypress/support/index.js

@@ -0,0 +1,26 @@
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+import "gatsby-cypress"
+import { addMatchImageSnapshotCommand } from "cypress-image-snapshot/command"
+
+addMatchImageSnapshotCommand({
+    customDiffDir: `/__diff_output__`,
+    customDiffConfig: {
+        threshold: 0.1
+    },
+    failureThreshold: 0.03,
+    failureThresholdType: `percent`
+})

e2e-tests/visual-regression/gatsby-config.js

@@ -0,0 +1,19 @@
+const path = require(`path`)
+
+module.exports = {
+  siteMetadata: {
+    title: `Gatsby Visual tests`,
+  },
+  plugins: [
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `img`,
+        path: `${__dirname}/src/images/`,
+      },
+    },
+    `gatsby-plugin-image`,
+    `gatsby-plugin-sharp`,
+    `gatsby-transformer-sharp`,
+  ],
+}

e2e-tests/visual-regression/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "gatsby-starter-default",
+  "description": "Gatsby default starter",
+  "version": "1.0.0",
+  "author": "Kyle Mathews <[email protected]>",
+  "dependencies": {
+    "cypress": "^3.1.0",
+    "cypress-image-snapshot": "^3.1.1",
+    "gatsby": "^2.0.118",
+    "gatsby-plugin-image": "^0.0.2",
+    "gatsby-plugin-sharp": "^2.0.20",
+    "gatsby-source-filesystem": "^2.3.35",
+    "gatsby-transformer-sharp": "^2.5.19",
+    "react": "^16.8.0",
+    "react-dom": "^16.8.0"
+  },
+  "keywords": [
+    "gatsby"
+  ],
+  "license": "MIT",
+  "scripts": {
+    "build": "gatsby build",
+    "develop": "gatsby develop",
+    "format": "prettier --write '**/*.js'",
+    "test": "cross-env CYPRESS_SUPPORT=y npm run build && npm run start-server-and-test",
+    "start-server-and-test": "start-server-and-test serve http://localhost:9000 cy:run",
+    "serve": "gatsby serve",
+    "cy:open": "cypress open",
+    "cy:run": "cypress run --browser chrome",
+    "cy:update-snapshots": "cypress run --browser chrome --env updateSnapshots=true",
+    "cy:clean-snapshots": "rimraf cypress/snapshots/*"
+  },
+  "devDependencies": {
+    "cross-env": "^5.2.0",
+    "cypress-junit-reporter": "^1.3.1",
+    "gatsby-cypress": "0.4.11",
+    "is-ci": "^2.0.0",
+    "prettier": "2.0.4",
+    "start-server-and-test": "^1.7.1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gatsbyjs/gatsby-starter-default"
+  }
+}

e2e-tests/visual-regression/src/components/header.js

@@ -0,0 +1,38 @@
+import * as React from "react"
+import { Link } from "gatsby"
+import PropTypes from "prop-types"
+
+const Header = ({ siteTitle }) => (
+  <div
+    style={{
+      background: `rebeccapurple`,
+      marginBottom: `1.45rem`,
+    }}
+  >
+    <div
+      style={{
+        margin: `0 auto`,
+        maxWidth: 960,
+        padding: `1.45rem 1.0875rem`,
+      }}
+    >
+      <h1 style={{ margin: 0 }}>
+        <Link
+          to="/"
+          style={{
+            color: `white`,
+            textDecoration: `none`,
+          }}
+        >
+          {siteTitle}
+        </Link>
+      </h1>
+    </div>
+  </div>
+)
+
+Header.propTypes = {
+  siteTitle: PropTypes.string.isRequired,
+}
+
+export default Header