Dramatically increase the complexity of everything

This was certainly one way to make sure the right hooks are firing when
we want. 

Particularly nasty is the use of `distanceFromTest` as a crude 
instrument to determine whether an afterEach is at a deeper depth than
an earlier error (to avoid running them unnecessarily). Just ugly as 
hell.

lib/build-test-actions.js

@@ -54,15 +54,13 @@ var actionsForTestFunctions = function (test, ancestors, helper, stats) {
     suiteNames: test.ancestorNames,
     children: _.flatten([
       actionForHook(helper.beforeEach, 'global', 'beforeEach', test.context, helper.file),
-      _.map(ancestors, function (ancestor) {
-        return actionForHook(ancestor.beforeEach, (ancestor.name || 'module'),
-                             'beforeEach', test.context, ancestor.file)
+      _.map(ancestors, function (ancestor, i) {
+        return actionForAncestorHook('beforeEach', ancestor, i, test)
       }),
       actionForTest(test, stats),
       actionForHook(helper.afterEach, 'global', 'afterEach', test.context, helper.file),
-      _(ancestors).reverse().map(function (ancestor) {
-        return actionForHook(ancestor.afterEach, (ancestor.name || 'module'),
-                             'afterEach', test.context, ancestor.file)
+      _(ancestors).reverse().map(function (ancestor, i) {
+        return actionForAncestorHook('afterEach', ancestor, i, test)
       }).value()
     ]),
     callable: function (cb) {
@@ -73,7 +71,12 @@ var actionsForTestFunctions = function (test, ancestors, helper, stats) {
   return testActions
 }
 
-var actionForHook = function (helperFunc, name, hookType, context, file) {
+function actionForAncestorHook (type, ancestor, distance, test) {
+  return actionForHook(ancestor[type], (ancestor.name || 'module'),
+                       type, test.context, ancestor.file, distance)
+}
+
+var actionForHook = function (helperFunc, name, hookType, context, file, distance) {
   var action = {
     name: name,
     type: 'userFunction',
@@ -82,7 +85,8 @@ var actionForHook = function (helperFunc, name, hookType, context, file) {
     description: 'test hook: ' + name + ' ' + hookType +
                  (file ? ' defined in `' + file + '`' : ''),
     callable: helperFunc.bind(context || {}),
-    isAssociatedWithATest: !_.isNil(context)
+    isAssociatedWithATest: !_.isNil(context),
+    distanceFromTest: distance || 0
   }
 
   return action
@@ -95,7 +99,8 @@ var actionForTest = function (test, stats) {
     subType: 'test',
     description: test.description(stats.suiteCount, stats.fileCounts[test.file]),
     callable: test.testFunction.bind(test.context),
-    isAssociatedWithATest: true
+    isAssociatedWithATest: true,
+    distanceFromTest: 0
   }
 
   return action

lib/plugins/internal/results.js

@@ -19,11 +19,13 @@ module.exports = function (log) {
       wrappers: {
         userFunction: function (runUserFunction, metadata, cb) {
           if (!shouldBeRun(results, metadata)) return cb(null)
+
           runUserFunction(function (er) {
             if (er) {
               markFailure(results, metadata, er)
               if (!metadata.isAssociatedWithATest) {
-                logError(log, metadata, _.last(results.nestedResults).errors.pop())
+                logError(log, metadata,
+                         _.last(_.last(results.nestedResults).errors))
               }
             }
             cb(null)
@@ -44,7 +46,9 @@ module.exports = function (log) {
             log((failed ? 'not ' : '') + 'ok ' + metadata.description)
 
             _.each(_.last(results.nestedResults).errors, function (er, i) {
-              logError(log, metadata, er)
+              if (er.metadata.isAssociatedWithATest) {
+                logError(log, metadata, er)
+              }
             })
 
             results.nestedResults.pop()
@@ -100,16 +104,21 @@ function logError (log, metadata, errorSummary) {
 
 function shouldBeRun (results, metadata) {
   if (!failedYet(results) ||
-      (testFuncErrored(results) && metadata.hookType === 'afterEach') ||
+      shouldRunAfterEach(results, metadata) ||
       metadata.hookType === 'afterAll') {
     return true
   } else {
     return false
   }
 }
 
-function testFuncErrored (results) {
+function shouldRunAfterEach (results, metadata) {
+  if (metadata.hookType !== 'afterEach') return false
+
   var errors = _.last(results.nestedResults).errors
-  return _.some(errors, 'metadata.isAssociatedWithATest')
+  return _(errors).map('metadata').some(function (errorMetadata) {
+    return errorMetadata.isAssociatedWithATest &&
+           errorMetadata.distanceFromTest <= metadata.distanceFromTest
+  })
 }
 

test/fixtures/reliable-hooks.js

@@ -23,6 +23,28 @@ module.exports = {
     test: mark('beforeAllFails.test'),
     afterEach: mark('beforeAllFails.afterEach'),
     afterAll: mark('beforeAllFails.afterAll')
+  },
+  beforeEachFails: {
+    nest: {
+      beforeAll: mark('beforeEachFails.nest.beforeAll'),
+      beforeEach: mark('beforeEachFails.nest.beforeEach'),
+      test: mark('beforeEachFails.nest.test'),
+      afterEach: mark('beforeEachFails.nest.afterEach'),
+      afterAll: mark('beforeEachFails.nest.afterAll')
+    },
+    beforeAll: mark('beforeEachFails.beforeAll'),
+    beforeEach: fail('beforeEachFails.beforeEach'),
+    test: mark('beforeEachFails.test'),
+    test2: mark('beforeEachFails.test2'),
+    afterEach: mark('beforeEachFails.afterEach'),
+    afterAll: mark('beforeEachFails.afterAll')
+  },
+  testFails: {
+    beforeAll: mark('testFails.beforeAll'),
+    beforeEach: mark('testFails.beforeEach'),
+    test: fail('testFails.test'),
+    afterEach: mark('testFails.afterEach'),
+    afterAll: mark('testFails.afterAll')
   }
 }
 

test/reliable-hooks-test.js

@@ -2,6 +2,7 @@ var helper = require('./support/helper')
 
 module.exports = function (cb) {
   helper.run('test/fixtures/reliable-hooks.js', function (er, result, log) {
+    // Without failure, it all runs (ofc)
     helper.deepEqual(global.__results.happy, {
       beforeAll: true,
       beforeEach: true,
@@ -10,6 +11,7 @@ module.exports = function (cb) {
       afterAll: true
     })
 
+    // If a beforeAll fails, only sibling afterAll runs, everything else stops
     helper.deepEqual(global.__results.beforeAllFails, {
       beforeAll: 'fail',
       beforeEach: false,
@@ -25,6 +27,32 @@ module.exports = function (cb) {
       }
     })
 
+    // If a beforeEach fails only sibling afterEach runs (the *Alls already ran)
+    helper.deepEqual(global.__results.beforeEachFails, {
+      beforeAll: true,
+      beforeEach: 'fail',
+      test: false,
+      test2: false,
+      afterEach: true,
+      afterAll: true,
+      nest: {
+        beforeAll: true,
+        beforeEach: false,
+        test: false,
+        afterEach: false,
+        afterAll: true
+      }
+    })
+
+    // If a test fails, everything else runs
+    helper.deepEqual(global.__results.testFails, {
+      beforeAll: true,
+      beforeEach: true,
+      test: 'fail',
+      afterEach: true,
+      afterAll: true
+    })
+
     delete global.__results
     cb(er)
   })