Fix runtime order for cases where beforeAll fails

In this case, afterAll should run but not the test or any subordinate
tests. In the case of outright-skipped test functions we print "SKIPPED"

lib/plugins/internal/results.js

@@ -18,9 +18,10 @@ module.exports = function (log) {
       sticky: false,
       wrappers: {
         userFunction: function (runUserFunction, metadata, cb) {
+          if (!shouldBeRun(results, metadata)) return cb(null)
           runUserFunction(function (er) {
             if (er) {
-              markFailure(results, er)
+              markFailure(results, metadata, er)
               if (!metadata.isAssociatedWithATest) {
                 logError(log, metadata, _.last(results.nestedResults).errors.pop())
               }
@@ -29,26 +30,33 @@ module.exports = function (log) {
           })
         },
         test: function (runTest, metadata, cb) {
+          if (failedYet(results)) {
+            log('not ok ' + metadata.description + ' [SKIPPED]')
+            return cb(null)
+          }
+          results.nestedResults.push({passing: true, errors: []})
           runTest(function (er) {
             if (er) {
-              markFailure(results, er)
+              markFailure(results, metadata, er)
             }
 
-            var failed = !_(results.nestedResults).map('passing').every()
+            var failed = failedYet(results)
             log((failed ? 'not ' : '') + 'ok ' + metadata.description)
 
             _.each(_.last(results.nestedResults).errors, function (er, i) {
               logError(log, metadata, er)
             })
 
+            results.nestedResults.pop()
             cb(null)
           })
         },
         suite: function (runSuite, metadata, cb) {
+          if (failedYet(results)) return cb(null)
           results.nestedResults.push({passing: true, errors: []})
           runSuite(function (er) {
             if (er) {
-              markFailure(results, er)
+              markFailure(results, metadata, er)
               logError(log, metadata, er)
             }
             results.nestedResults.pop()
@@ -63,14 +71,22 @@ module.exports = function (log) {
   }
 }
 
-function markFailure (results, er) {
+function markFailure (results, metadata, er) {
   results.overallPassing = false
   var currentNesting = _.last(results.nestedResults)
   currentNesting.passing = false
-  currentNesting.errors.push(er)
+  currentNesting.errors.push({
+    metadata: metadata,
+    error: er
+  })
 }
 
-function logError (log, metadata, e) {
+function failedYet (results) {
+  return !_(results.nestedResults).map('passing').every()
+}
+
+function logError (log, metadata, errorSummary) {
+  var e = errorSummary.error
   if (e) {
     if (metadata.type === 'userFunction' && metadata.subType === 'hook') {
       log(' An error occurred in ' + metadata.description)
@@ -81,3 +97,19 @@ function logError (log, metadata, e) {
     log('  ...')
   }
 }
+
+function shouldBeRun (results, metadata) {
+  if (!failedYet(results) ||
+      (testFuncErrored(results) && metadata.hookType === 'afterEach') ||
+      metadata.hookType === 'afterAll') {
+    return true
+  } else {
+    return false
+  }
+}
+
+function testFuncErrored (results) {
+  var errors = _.last(results.nestedResults).errors
+  return _.some(errors, 'metadata.isAssociatedWithATest')
+}
+

test/async-test.js

@@ -12,8 +12,8 @@ module.exports = function (cb) {
       '  message: nope',
       /stacktrace: Error: nope/,
       '  ...',
-      'not ok 1 - "test1" - test #1 in `test/fixtures/async-beforeAll-fails-all.js`',
-      'not ok 2 - "test2" - test #2 in `test/fixtures/async-beforeAll-fails-all.js`',
+      'not ok 1 - "test1" - test #1 in `test/fixtures/async-beforeAll-fails-all.js` [SKIPPED]',
+      'not ok 2 - "test2" - test #2 in `test/fixtures/async-beforeAll-fails-all.js` [SKIPPED]',
       'not ok 3 - test #1 in `test/fixtures/async-error-test.js`',
       '  ---',
       '  message: Something bad',

test/fixtures/reliable-hooks.js

@@ -0,0 +1,42 @@
+var _ = require('lodash')
+global.__results = {}
+
+// This is a dye test to verify that all the expected hooks ran (or didn't)
+module.exports = {
+  happy: {
+    beforeAll: mark('happy.beforeAll'),
+    beforeEach: mark('happy.beforeEach'),
+    test: mark('happy.test'),
+    afterEach: mark('happy.afterEach'),
+    afterAll: mark('happy.afterAll')
+  },
+  beforeAllFails: {
+    nest: {
+      beforeAll: mark('beforeAllFails.nest.beforeAll'),
+      beforeEach: mark('beforeAllFails.nest.beforeEach'),
+      test: mark('beforeAllFails.nest.test'),
+      afterEach: mark('beforeAllFails.nest.afterEach'),
+      afterAll: mark('beforeAllFails.nest.afterAll')
+    },
+    beforeAll: fail('beforeAllFails.beforeAll'),
+    beforeEach: mark('beforeAllFails.beforeEach'),
+    test: mark('beforeAllFails.test'),
+    afterEach: mark('beforeAllFails.afterEach'),
+    afterAll: mark('beforeAllFails.afterAll')
+  }
+}
+
+function mark (path) {
+  _.set(global.__results, path, false)
+  return function () {
+    _.set(global.__results, path, true)
+  }
+}
+
+function fail (path) {
+  _.set(global.__results, path, false)
+  return function () {
+    _.set(global.__results, path, 'fail')
+    throw new Error('Intentional failure at ' + path)
+  }
+}

test/reliable-hooks-test.js

@@ -0,0 +1,32 @@
+var helper = require('./support/helper')
+
+module.exports = function (cb) {
+  helper.run('test/fixtures/reliable-hooks.js', function (er, result, log) {
+    helper.deepEqual(global.__results.happy, {
+      beforeAll: true,
+      beforeEach: true,
+      test: true,
+      afterEach: true,
+      afterAll: true
+    })
+
+    helper.deepEqual(global.__results.beforeAllFails, {
+      beforeAll: 'fail',
+      beforeEach: false,
+      test: false,
+      afterEach: false,
+      afterAll: true,
+      nest: {
+        beforeAll: false,
+        beforeEach: false,
+        test: false,
+        afterEach: false,
+        afterAll: false
+      }
+    })
+
+    delete global.__results
+    cb(er)
+  })
+}
+

test/support/helper.js

@@ -20,7 +20,7 @@ module.exports = {
   deepEqual: function (actual, expected, msg) {
     try {
       assert.deepEqual(actual, expected, msg)
-    } catch(e) {
+    } catch (e) {
       console.log('Failed comparing actual:')
       console.log(actual)
       console.log('with expected:')