Set up CircleCI test summary [npmcollaborators] (#2526)

See https://circleci.com/docs/2.0/collect-test-data/ This provides a list of failed tests in the Circle UI and a summary of which tests have failed. It should make interpreting test results easier
badges · Dec 16, 2018 · 2bf29f9 · 2bf29f9
1 parent b5ac5d6
commit 2bf29f9
Show file tree

Hide file tree

Showing 4 changed files with 150 additions and 32 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -9,9 +9,9 @@ jobs:
 
       - restore_cache:
           keys:
-          - v2-dependencies-{{ checksum "package.json" }}
-          # fallback to using the latest cache if no exact match is found
-          - v2-dependencies-
+            - v2-dependencies-{{ checksum "package.json" }}
+            # fallback to using the latest cache if no exact match is found
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -30,9 +30,10 @@ jobs:
       - checkout
 
       - restore_cache:
-          key: v2-dependencies-{{ checksum "package.json" }}
-          # https://github.com/badges/shields/issues/1937
-          key: v2-dependencies-
+          keys:
+            - v2-dependencies-{{ checksum "package.json" }}
+            # https://github.com/badges/shields/issues/1937
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -43,21 +44,38 @@ jobs:
           when: always
           command: npm run lint
 
+      - run:
+          name: Prepare test summary
+          when: always
+          command: mkdir junit
+
       - run:
           name: Server unit tests
           when: always
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/server/results.xml
           command: npm run test:js:server
 
       - run:
           name: Integration tests
           when: always
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/server/integration.xml
           command: npm run test:integration
 
       - run:
           name: Tests for gh-badges package
           when: always
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/server/gh-badges.xml
           command: npm run test:js:package
 
+      - store_test_results:
+          path: junit
+
       - run:
           name: 'Prettier check (quick fix: `npm run prettier`)'
           when: always
@@ -71,9 +89,10 @@ jobs:
       - checkout
 
       - restore_cache:
-          key: v2-dependencies-{{ checksum "package.json" }}
-          # https://github.com/badges/shields/issues/1937
-          key: v2-dependencies-
+          keys:
+            - v2-dependencies-{{ checksum "package.json" }}
+            # https://github.com/badges/shields/issues/1937
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -84,21 +103,38 @@ jobs:
           when: always
           command: npm run lint
 
+      - run:
+          name: Prepare test summary
+          when: always
+          command: mkdir junit
+
       - run:
           name: Server unit tests
           when: always
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/server/results.xml
           command: npm run test:js:server
 
       - run:
           name: Integration tests
           when: always
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/integration/results.xml
           command: npm run test:integration
 
       - run:
           name: Tests for gh-badges package
           when: always
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/gh-badges/results.xml
           command: npm run test:js:package
 
+      - store_test_results:
+          path: junit
+
       - run:
           name: 'Prettier check (quick fix: `npm run prettier`)'
           when: always
@@ -111,9 +147,10 @@ jobs:
       - checkout
 
       - restore_cache:
-          key: v2-dependencies-{{ checksum "package.json" }}
-          # https://github.com/badges/shields/issues/1937
-          key: v2-dependencies-
+          keys:
+            - v2-dependencies-{{ checksum "package.json" }}
+            # https://github.com/badges/shields/issues/1937
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -131,9 +168,10 @@ jobs:
       - checkout
 
       - restore_cache:
-          key: v2-dependencies-{{ checksum "package.json" }}
-          # https://github.com/badges/shields/issues/1937
-          key: v2-dependencies-
+          keys:
+            - v2-dependencies-{{ checksum "package.json" }}
+            # https://github.com/badges/shields/issues/1937
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -143,10 +181,21 @@ jobs:
           name: Prepare frontend tests
           command: npm run defs && npm run features
 
+      - run:
+          name: Prepare test summary
+          when: always
+          command: mkdir junit
+
       - run:
           name: Frontend unit tests
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/frontend/results.xml
           command: npm run test:js:frontend
 
+      - store_test_results:
+          path: junit
+
       - run:
           name: Frontend build completes successfully
           when: always
@@ -165,9 +214,10 @@ jobs:
             echo "{\"gh_token\":\"$GITHUB_TOKEN\", \"wheelmap_token\":\"$WHEELMAP_TOKEN\"}" > private/secret.json
 
       - restore_cache:
-          key: v2-dependencies-{{ checksum "package.json" }}
-          # https://github.com/badges/shields/issues/1937
-          key: v2-dependencies-
+          keys:
+            - v2-dependencies-{{ checksum "package.json" }}
+            # https://github.com/badges/shields/issues/1937
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -177,10 +227,21 @@ jobs:
           name: Identify services tagged in the PR title
           command: npm run test:services:pr:prepare
 
+      - run:
+          name: Prepare test summary
+          when: always
+          command: mkdir junit
+
       - run:
           name: Run tests for tagged services
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/services/results.xml
           command: npm run test:services:pr:run
 
+      - store_test_results:
+          path: junit
+
   services@node-latest:
     docker:
       - image: circleci/node:latest
@@ -194,9 +255,10 @@ jobs:
             echo "{\"gh_token\":\"$GITHUB_TOKEN\"}" > private/secret.json
 
       - restore_cache:
-          key: v2-dependencies-{{ checksum "package.json" }}
-          # https://github.com/badges/shields/issues/1937
-          key: v2-dependencies-
+          keys:
+            - v2-dependencies-{{ checksum "package.json" }}
+            # https://github.com/badges/shields/issues/1937
+            - v2-dependencies-
 
       - run:
           name: Install dependencies
@@ -206,10 +268,21 @@ jobs:
           name: Identify services tagged in the PR title
           command: npm run test:services:pr:prepare
 
+      - run:
+          name: Prepare test summary
+          when: always
+          command: mkdir junit
+
       - run:
           name: Run tests for tagged services
+          environment:
+            mocha_reporter: mocha-junit-reporter
+            MOCHA_FILE: junit/services/results.xml
           command: npm run test:services:pr:run
 
+      - store_test_results:
+          path: junit
+
 workflows:
   version: 2
 

diff --git a/mocha.opts b/mocha.opts
@@ -0,0 +1 @@
+--reporter mocha-env-reporter
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -58,11 +58,11 @@
     "xpath": "~0.0.27"
   },
   "scripts": {
-    "coverage:test:frontend": "BABEL_ENV=test nyc node_modules/mocha/bin/_mocha --require @babel/polyfill --require @babel/register --require mocha-yaml-loader \"frontend/**/*.spec.js\"",
-    "coverage:test:server": "HANDLE_INTERNAL_ERRORS=false nyc node_modules/mocha/bin/_mocha \"*.spec.js\" \"lib/**/*.spec.js\" \"services/**/*.spec.js\"",
-    "coverage:test:package": "nyc node_modules/mocha/bin/_mocha \"gh-badges/**/*.spec.js\"",
-    "coverage:test:integration": "nyc node_modules/mocha/bin/_mocha \"lib/**/*.integration.js\" \"services/**/*.integration.js\"",
-    "coverage:test:services": "nyc node_modules/mocha/bin/_mocha --delay lib/service-test-runner/cli.js",
+    "coverage:test:frontend": "BABEL_ENV=test nyc node_modules/mocha/bin/_mocha --opts mocha.opts --require @babel/polyfill --require @babel/register --require mocha-yaml-loader \"frontend/**/*.spec.js\"",
+    "coverage:test:server": "HANDLE_INTERNAL_ERRORS=false nyc node_modules/mocha/bin/_mocha --opts mocha.opts \"*.spec.js\" \"lib/**/*.spec.js\" \"services/**/*.spec.js\"",
+    "coverage:test:package": "nyc node_modules/mocha/bin/_mocha --opts mocha.opts \"gh-badges/**/*.spec.js\"",
+    "coverage:test:integration": "nyc node_modules/mocha/bin/_mocha --opts mocha.opts \"lib/**/*.integration.js\" \"services/**/*.integration.js\"",
+    "coverage:test:services": "nyc node_modules/mocha/bin/_mocha --opts mocha.opts --delay lib/service-test-runner/cli.js",
     "coverage:test": "rimraf .nyc_output coverage; npm run coverage:test:server; npm run coverage:test:package; npm run coverage:test:frontend; npm run coverage:test:integration; npm run coverage:test:services",
     "coverage:report": "nyc report",
     "coverage:report:reopen": "opn coverage/lcov-report/index.html",
@@ -71,14 +71,14 @@
     "prettier": "prettier --write \"**/*.js\"",
     "prettier-check": "prettier-check \"**/*.js\"",
     "danger": "danger",
-    "test:js:frontend": "BABEL_ENV=test mocha --require @babel/polyfill --require @babel/register --require mocha-yaml-loader \"frontend/**/*.spec.js\"",
-    "test:js:server": "HANDLE_INTERNAL_ERRORS=false mocha \"*.spec.js\" \"lib/**/*.spec.js\" \"services/**/*.spec.js\"",
-    "test:js:package": "mocha \"gh-badges/**/*.spec.js\"",
-    "test:integration": "mocha \"lib/**/*.integration.js\" \"services/**/*.integration.js\"",
-    "test:services": "HANDLE_INTERNAL_ERRORS=false mocha --delay lib/service-test-runner/cli.js",
+    "test:js:frontend": "BABEL_ENV=test mocha --opts mocha.opts --require @babel/polyfill --require @babel/register --require mocha-yaml-loader \"frontend/**/*.spec.js\"",
+    "test:js:server": "HANDLE_INTERNAL_ERRORS=false mocha --opts mocha.opts \"*.spec.js\" \"lib/**/*.spec.js\" \"services/**/*.spec.js\"",
+    "test:js:package": "mocha --opts mocha.opts \"gh-badges/**/*.spec.js\"",
+    "test:integration": "mocha --opts mocha.opts \"lib/**/*.integration.js\" \"services/**/*.integration.js\"",
+    "test:services": "HANDLE_INTERNAL_ERRORS=false mocha --opts mocha.opts --delay lib/service-test-runner/cli.js",
     "test:services:trace": "TRACE_SERVICES=true npm run test:services -- $*",
     "test:services:pr:prepare": "node lib/service-test-runner/pull-request-services-cli.js > pull-request-services.log",
-    "test:services:pr:run": "HANDLE_INTERNAL_ERRORS=false mocha --delay lib/service-test-runner/cli.js --stdin < pull-request-services.log",
+    "test:services:pr:run": "HANDLE_INTERNAL_ERRORS=false mocha --opts mocha.opts --delay lib/service-test-runner/cli.js --stdin < pull-request-services.log",
     "test:services:pr": "npm run test:services:pr:prepare && npm run test:services:pr:run",
     "test": "npm run defs && npm run lint && npm run test:js:frontend && npm run test:js:package && npm run test:js:server",
     "depcheck": "check-node-version --node \">= 8.0\"",
@@ -157,6 +157,8 @@
     "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
     "mocha": "^5.0.0",
+    "mocha-env-reporter": "^3.0.0",
+    "mocha-junit-reporter": "^1.18.0",
     "mocha-yaml-loader": "^1.0.3",
     "next": "^6.1.1",
     "nock": "^10.0.4",