Integrate JsReporters

lib/_patch/reporter.js

@@ -0,0 +1,48 @@
+(function() {
+  var runner;
+  var total = 0,
+      passed = 0,
+      failed = 0;
+
+  if (window.QUnit) {
+    runner = new JsReporters.QUnitAdapter(QUnit);
+  } else if (window.jasmine) {
+    runner = new JsReporters.JasmineAdapter(jasmine.getEnv());
+  } else if (window.mocha) {
+    runner = new JsReporters.MochaAdapter(mocha);
+  } else {
+    throw new Error('JsReporters: No testing framework was found');
+  }
+
+  runner.on('testEnd', function(test) {
+    total = total + 1
+
+    passed = passed + (test.status === 'passed' ? 1 : 0);
+    failed = failed + (test.status === 'failed' ? 1 : 0);
+
+    test.errors.forEach(function(error) {
+      BrowserStack.post("/_progress", {
+        tracebacks: [{
+          actual: error.actual,
+          expected: error.expected,
+          message: error.message,
+          source: error.source || error.stack,
+          testName: test.testName
+        }]
+      }, function() {});
+    });
+  });
+
+  runner.on('runEnd', function(globalSuite) {
+    var results = {};
+
+    results.runtime = globalSuite.runtime;
+    results.total = total;
+    results.passed = passed;
+    results.failed = failed;
+    results.url = window.location.pathname;
+
+    BrowserStack.post("/_report", results, function() {});
+  });
+})();
+

lib/server.js

@@ -39,15 +39,17 @@ exports.Server = function Server(bsClient, workers) {
       ];
 
       var framework_scripts = {
-        'qunit': ['qunit-plugin.js'],
-        'jasmine': ['jasmine-jsreporter.js', 'jasmine-plugin.js'],
-        'jasmine2': ['jasmine2-plugin.js'],
-        'mocha': ['mocha-plugin.js']
+        'jasmine': ['jasmine-jsreporter.js', 'jasmine-plugin.js']
       };
 
       var filePath = path.relative(process.cwd(), filename);
       var pathMatches = (testFilePaths.indexOf(filePath) !== -1);
 
+      var jsReportersPath = path.join(__dirname, '../node_modules/js-reporters/dist/js-reporters.js');
+      var jsReportersScript = fs.readFileSync(jsReportersPath, {
+        encoding: 'utf8'
+      });
+
       if (pathMatches) {
         var framework = config['test_framework'];
         var tag_name = (framework === 'mocha') ? 'head' : 'body';
@@ -57,25 +59,16 @@ exports.Server = function Server(bsClient, workers) {
           patch += '<script type="text/javascript" src="/_patch/' + script + '"></script>\n';
         });
 
+        patch += '<script type="text/javascript">' + jsReportersScript + '</script>';
+
         // adding framework scripts
         if (framework === 'jasmine') {
           framework_scripts['jasmine'].forEach(function(script) {
             patch += '<script type="text/javascript" src="/_patch/' + script + '"></script>\n';
           });
           patch += '<script type="text/javascript">jasmine.getEnv().addReporter(new jasmine.JSReporter());</script>\n';
-        } else if (framework === 'jasmine2') {
-          framework_scripts['jasmine2'].forEach(function(script) {
-            patch += '<script type="text/javascript" src="/_patch/' + script + '"></script>\n';
-          });
-        } else if (framework === 'mocha') {
-          framework_scripts['mocha'].forEach(function(script) {
-            patch += '<script type="text/javascript" src="/_patch/' + script + '"></script>\n';
-          });
-          patch += '<script type="text/javascript">mocha.reporter(Mocha.BrowserStack);</script>\n';
-        } else if (framework === 'qunit') {
-          framework_scripts['qunit'].forEach(function(script) {
-            patch += '<script type="text/javascript" src="/_patch/' + script + '"></script>\n';
-          });
+        } else {
+          patch += '<script type="text/javascript" src="/_patch/reporter.js"></script>\n';
         }
         patch += '</' + tag_name + '>';
         return patch;

package.json

@@ -10,6 +10,7 @@
   "dependencies": {
     "browserstack": "1.3.0",
     "chalk": "0.4.0",
+    "js-reporters": "^1.0.0",
     "mime": "1.3.4",
     "send": "0.13.0",
     "tunnel": "0.0.3"

tests/external-tests.js

@@ -13,6 +13,47 @@ var runnerPath = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js'));
 var testHome = path.resolve(__dirname);
 process.chdir(testHome);
 
+/**
+ * Mocha v2.4.5 - to change with another Mocha version or
+ * something with Mocha tests
+ *
+ * index.html - 22 tests, 18 passed, 4 failed -> one test is displayed twice,
+ *  so they are displayed 5 failing tests, but counted only 4
+ * large.html - 64 tests, 60 passed, 4 failed -> only 2 tests are failing, but
+ *  they are displayed twice
+ * opts.html - 8 tests, 2 passed, 6 failed -> only 3 tests are failing, but
+ *  they are displayed twice
+ *
+ * By "displayed" it is referred the Mocha HTML Reporter.
+ *
+ * From the above explanations it is clear that there are some inconsistencies,
+ * also because Mocha's HTML Reporter counted number of tests does not match
+ * the number of displyed tests.
+ *
+ * The cause is (snippet from Mocha's HTML reporter):
+ *
+ * runner.on('fail', function(test) {
+ *  // For type = 'test' its possible that the test failed due to multiple
+ *  // done() calls. So report the issue here.
+ *  if (test.type === 'hook'
+ *    || test.type === 'test') {
+ *    runner.emit('test end', test);
+ *  }
+ * });
+ *
+ * This is why failed tests are displayed twice...
+ *
+ * The JsReporters is counting the tests on the "test end" event, that's why
+ * it is capturing the failing tests twice, in the "index.html" it does not
+ * capture everything, because there is an async test, which failure is
+ * triggered after a timeout and the JsReporters is not waiting, because
+ * it cannot know how much to wait.
+ *
+ *
+ * This been said, the JsReporter MochaAdapter is functioning well, this
+ * version of Mocha is not reliable and should be changed.
+ */
+
 var repositories = [
   {
     name: 'qunit',
@@ -31,8 +72,8 @@ var repositories = [
       'test/index.html'
     ],
     expected_results: {
-      tests: 534,
-      passed: 534,
+      tests: 133,
+      passed: 130,
       failed: 0
     }
   },
@@ -55,9 +96,9 @@ var repositories = [
       'test/browser/opts.html'
     ],
     expected_results: {
-      tests: 89,
+      tests: 94,
       passed: 80,
-      failed: 9
+      failed: 14
     }
   },
   {
@@ -113,29 +154,6 @@ var repositories = [
 ];
 
 var repositoriesOptional = [
-  {
-    name: 'qunit',
-    tag: 'v1.0.0',
-    url: 'https://github.com/jquery/qunit.git',
-    test_framework: 'qunit',
-    browsers: [
-      {
-        'browser': 'firefox',
-        'browser_version': '44.0',
-        'os': 'OS X',
-        'os_version': 'Snow Leopard'
-      }
-    ],
-    test_path: [
-      'test/index.html',
-      'test/logs.html'
-    ],
-    expected_results: {
-      tests: 323,
-      passed: 323,
-      failed: 0
-    }
-  },
   {
     name: 'mocha',
     tag: '1.21.5',
@@ -155,9 +173,9 @@ var repositoriesOptional = [
       'test/browser/opts.html'
     ],
     expected_results: {
-      tests: 84,
+      tests: 83,
       passed: 77,
-      failed: 7
+      failed: 6
     }
   }
 ];