Added option cliExitOnEngineError to bubble up if engine has an error…

… e.g. missing selector

README.md

@@ -571,6 +571,12 @@ or
 ```json
 "engine": "chromy"
 ```
+### Exposing engine errors
+By default if there is an error in the engine e.g. missing selector, it won't exit BackstopJS. This is normally fine as the screenshots will pick up any unexpected differences. If you want to terminate BackstopJS on engine errors set the option flag:
+
+```json
+"cliExitOnEngineError": true
+```
 
 ### Setting Puppeteer option flags
 Backstop sets two defaults for Puppeteer:

core/util/runChromy.js

@@ -2,6 +2,7 @@ const Chromy = require('chromy');
 // const writeFileSync = require('fs').writeFileSync;
 const fs = require('./fs');
 const path = require('path');
+const chalk = require('chalk');
 const ensureDirectoryPath = require('./ensureDirectoryPath');
 const engineTools = require('./engineTools');
 
@@ -319,8 +320,15 @@ function processScenarioView (scenario, variantOrScenarioLabelSafe, scenarioLabe
         ));
       })
       .end()
-      // If an error occurred then resolve with an error.
-      .catch(e => resolve(new BackstopException('Chromy error', scenario, viewport, e)));
+      .catch(e => {
+        const err = new BackstopException('Chromy error', scenario, viewport, e);
+        console.log(chalk.red(err));
+        if (config.cliExitOnEngineError) {
+          chromy.close().end();
+          return reject(err);
+        }
+        return resolve(err);
+      });
   });
 }
 

core/util/runPuppet.js

@@ -22,6 +22,8 @@ const DOCUMENT_SELECTOR = 'document';
 const NOCLIP_SELECTOR = 'body:noclip';
 const VIEWPORT_SELECTOR = 'viewport';
 
+const BackstopException = require('../util/BackstopException.js');
+
 module.exports = function (args) {
   const scenario = args.scenario;
   const viewport = args.viewport;
@@ -224,8 +226,7 @@ async function processScenarioView (scenario, variantOrScenarioLabelSafe, scenar
 
   let error;
   await puppetCommands().catch(e => {
-    console.log(chalk.red(`Puppeteer encountered an error while running scenario "${scenario.label}"`));
-    console.log(chalk.red(e));
+    console.log(chalk.red(new BackstopException('Puppeteer error', scenario, viewport, e)));
     error = e;
   });
 
@@ -259,6 +260,10 @@ async function processScenarioView (scenario, variantOrScenarioLabelSafe, scenar
       testPairs: [ testPair ]
     };
     fs.copy(config.env.backstop + ERROR_SELECTOR_PATH, filePath);
+
+    if (config.cliExitOnEngineError) {
+      return Promise.reject(compareConfig);
+    }
   }
 
   return Promise.resolve(compareConfig);

examples/responsiveDemo/package.json

@@ -5,6 +5,8 @@
   "main": "index.html",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
+    "dev-backstop-test": "node ../../cli/index.js test",
+    "dev-backstop-reference": "node ../../cli/index.js reference",
     "serve": "node ./node_modules/super-simple-web-server/ ./"
   },
   "author": "",