[ftr] add ability to get included/excluded test counts

src/functional_test_runner/cli.js

@@ -48,6 +48,7 @@ cmd
   .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), [])
   .option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), [])
   .option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), [])
+  .option('--test-stats', 'Print the number of tests (included and excluded) to STDERR', false)
   .option('--verbose', 'Log everything', false)
   .option('--quiet', 'Only log errors', false)
   .option('--silent', 'Log nothing', false)
@@ -86,8 +87,16 @@ const functionalTestRunner = createFunctionalTestRunner({
 
 async function run() {
   try {
-    const failureCount = await functionalTestRunner.run();
-    process.exitCode = failureCount ? 1 : 0;
+    if (cmd.testStats) {
+      process.stderr.write(JSON.stringify(
+        await functionalTestRunner.getTestStats(),
+        null,
+        2
+      ) + '\n');
+    } else {
+      const failureCount = await functionalTestRunner.run();
+      process.exitCode = failureCount ? 1 : 0;
+    }
   } catch (err) {
     await teardown(err);
   } finally {

src/functional_test_runner/functional_test_runner.js

@@ -20,7 +20,8 @@
 import {
   createLifecycle,
   readConfigFile,
-  createProviderCollection,
+  ProviderCollection,
+  readProviderSpec,
   setupMocha,
   runTests,
 } from './lib';
@@ -36,8 +37,61 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
     log.verbose('ending %j lifecycle phase', name);
   });
 
+
   class FunctionalTestRunner {
     async run() {
+      return await this._run(async (config, coreProviders) => {
+        const providers = new ProviderCollection(log, [
+          ...coreProviders,
+          ...readProviderSpec('Service', config.get('services')),
+          ...readProviderSpec('PageObject', config.get('pageObjects'))
+        ]);
+
+        await providers.loadAll();
+
+        const mocha = await setupMocha(lifecycle, log, config, providers);
+        await lifecycle.trigger('beforeTests');
+        log.info('Starting tests');
+
+        return await runTests(lifecycle, log, mocha);
+      });
+    }
+
+    async getTestStats() {
+      return await this._run(async (config, coreProviders) => {
+        // replace the function of a provider so that it returns a promise-like object which
+        // never "resolves", essentially disabling all the services while still allowing us
+        // to load the test files and populate the mocha suites
+        const stubProvider = provider => ({
+          ...provider,
+          fn: () => ({
+            then: () => {}
+          })
+        });
+
+        const providers = new ProviderCollection(log, [
+          ...coreProviders,
+          ...readProviderSpec('Service', config.get('services')),
+          ...readProviderSpec('PageObject', config.get('pageObjects'))
+        ].map(stubProvider));
+
+        const mocha = await setupMocha(lifecycle, log, config, providers);
+
+        const countTests = suite => (
+          suite.suites.reduce(
+            (sum, suite) => sum + countTests(suite),
+            suite.tests.length
+          )
+        );
+
+        return {
+          tests: countTests(mocha.suite),
+          excludedTests: mocha.excludedTests.length
+        };
+      });
+    }
+
+    async _run(handler) {
       let runErrorOccurred = false;
 
       try {
@@ -49,14 +103,14 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
           return;
         }
 
-        const providers = createProviderCollection(lifecycle, log, config);
-        await providers.loadAll();
-
-        const mocha = await setupMocha(lifecycle, log, config, providers);
-        await lifecycle.trigger('beforeTests');
-        log.info('Starting tests');
-        return await runTests(lifecycle, log, mocha);
+        // base level services that functional_test_runner exposes
+        const coreProviders = readProviderSpec('Service', {
+          lifecycle: () => lifecycle,
+          log: () => log,
+          config: () => config,
+        });
 
+        return await handler(config, coreProviders);
       } catch (runError) {
         runErrorOccurred = true;
         throw runError;

src/functional_test_runner/lib/index.js

@@ -19,5 +19,5 @@
 
 export { createLifecycle } from './lifecycle';
 export { readConfigFile } from './config';
-export { createProviderCollection } from './create_provider_collection';
 export { setupMocha, runTests } from './mocha';
+export { readProviderSpec, ProviderCollection } from './providers';

src/functional_test_runner/lib/mocha/filter_suites_by_tags.js

@@ -28,6 +28,14 @@
  * @param options.exclude an array of tags that will be used to exclude suites from the run
  */
 export function filterSuitesByTags({ log, mocha, include, exclude }) {
+  mocha.excludedTests = [];
+  // collect all the tests from some suite, including it's children
+  const collectTests = (suite) =>
+    suite.suites.reduce(
+      (acc, s) => acc.concat(collectTests(s)),
+      suite.tests
+    );
+
   // if include tags were provided, filter the tree once to
   // only include branches that are included at some point
   if (include.length) {
@@ -47,14 +55,18 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
           continue;
         }
 
+
         // this suite has an included child but is not included
         // itself, so strip out its tests and recurse to filter
         // out child suites which are not included
         if (isChildIncluded(child)) {
+          mocha.excludedTests = mocha.excludedTests.concat(child.tests);
           child.tests = [];
           parentSuite.suites.push(child);
           recurse(child);
           continue;
+        } else {
+          mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
         }
       }
     }(mocha.suite));
@@ -78,6 +90,8 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
         if (isNotExcluded(child)) {
           parentSuite.suites.push(child);
           recurse(child);
+        } else {
+          mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
         }
       }
     }(mocha.suite));