Taskcluster CI/CD workflow.

.dockerignore

@@ -0,0 +1,4 @@
+.git
+frontend/
+addon/
+*/tests/

.taskcluster.yml

@@ -0,0 +1,344 @@
+version: 1
+policy:
+  pullRequests: public
+tasks:
+  $let:
+    head_branch:
+      $if: 'tasks_for == "github-pull-request"'
+      then: ${event.pull_request.head.ref}
+      else:
+        $if: 'tasks_for == "github-push"'
+        then: ${event.ref}
+        else: ${event.release.target_commitish}
+
+    head_rev:
+      $if: 'tasks_for == "github-pull-request"'
+      then: ${event.pull_request.head.sha}
+      else:
+        $if: 'tasks_for == "github-push"'
+        then: ${event.after}
+        else: ${event.release.tag_name}
+
+    repository:
+      $if: 'tasks_for == "github-pull-request"'
+      then: ${event.pull_request.head.repo.html_url}
+      else: ${event.repository.html_url}
+
+    channel:
+      $if: 'tasks_for == "github-push"'
+      then:
+        $if: 'event.ref in ["refs/heads/testing", "refs/heads/production"]'
+        then: ${event.ref[11:]}
+        else: 'dev'
+      else: 'dev'
+
+    taskboot_image: "mozilla/taskboot:0.1.9"
+  in:
+    - taskId: {$eval: as_slugid("bot_check_lint")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: python:3
+        command:
+          - sh
+          - -lxce
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/tools && python setup.py install &&
+            cd /src/bot && pip install --quiet . && pip install --quiet -r requirements-dev.txt &&
+            flake8"
+      metadata:
+        name: "Code Coverage Bot checks: linting"
+        description: Check python code style with flake8
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("backend_check_lint")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: python:3
+        command:
+          - sh
+          - -lxce
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/tools && python setup.py install &&
+            cd /src/backend && pip install --quiet . && pip install --quiet -r requirements-dev.txt &&
+            flake8"
+      metadata:
+        name: "Code Coverage Backend checks: linting"
+        description: Check python code style with flake8
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("bot_check_tests")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: python:3
+        command:
+          - sh
+          - -lxce
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            /src/bot/ci/bootstrap.sh &&
+            cd /src/tools && python3 setup.py install &&
+            cd /src/bot && pip3 install --quiet . && pip3 install --quiet -r requirements-dev.txt &&
+            pytest -v"
+      metadata:
+        name: "Code Coverage Bot checks: unit tests"
+        description: Check python code with pytest
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("backend_check_tests")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: python:3
+        command:
+          - sh
+          - -lxce
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
+            cd /src/tools && python setup.py install &&
+            cd /src/backend && pip install --quiet . && pip install --quiet -r requirements-dev.txt &&
+            pytest -v"
+      metadata:
+        name: "Code Coverage Backend checks: unit tests"
+        description: Check python code with pytest
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("backend_build")}
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      provisionerId: aws-provisioner-v1
+      workerType: releng-svc
+      dependencies:
+        - {$eval: as_slugid("backend_check_lint")}
+        - {$eval: as_slugid("backend_check_tests")}
+      payload:
+        capabilities:
+          privileged: true
+        maxRunTime: 3600
+        image: "${taskboot_image}"
+        env:
+          GIT_REPOSITORY: ${repository}
+          GIT_REVISION: ${head_rev}
+        command:
+          - taskboot
+          - build
+          - --image
+          - mozilla/code-coverage/backend
+          - --tag
+          - "${channel}"
+          - --tag
+          - "${head_rev}"
+          - --write
+          - /backend.tar
+          - backend/Dockerfile
+        artifacts:
+          public/code-coverage-backend.tar:
+            expires: {$fromNow: '2 weeks'}
+            path: /backend.tar
+            type: file
+      scopes:
+        - docker-worker:capability:privileged
+      metadata:
+        name: Code Coverage Backend docker build
+        description: Build docker image with taskboot
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("bot_build")}
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      provisionerId: aws-provisioner-v1
+      workerType: releng-svc
+      dependencies:
+        - {$eval: as_slugid("bot_check_lint")}
+        - {$eval: as_slugid("bot_check_tests")}
+      payload:
+        capabilities:
+          privileged: true
+        maxRunTime: 3600
+        image: "${taskboot_image}"
+        env:
+          GIT_REPOSITORY: ${repository}
+          GIT_REVISION: ${head_rev}
+        command:
+          - taskboot
+          - build
+          - --image
+          - mozilla/code-coverage
+          - --tag
+          - "bot-${channel}"
+          - --tag
+          - "bot-${head_rev}"
+          - --write
+          - /bot.tar
+          - bot/Dockerfile
+        artifacts:
+          public/code-coverage-bot.tar:
+            expires: {$fromNow: '2 weeks'}
+            path: /bot.tar
+            type: file
+      scopes:
+        - docker-worker:capability:privileged
+      metadata:
+        name: Code Coverage Bot docker build
+        description: Build docker image with taskboot
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("addon_build")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: node:11-alpine
+        command:
+          - sh
+          - -lxce
+          - "apk add --quiet --update git python &&
+            git clone --quiet ${repository} /src && cd /src/addon && git checkout ${head_rev} -b build &&
+            npm install --no-progress &&
+            npm run build &&
+            npm run test &&
+            npm run export"
+        artifacts:
+          public/addon:
+            expires: {$fromNow: '2 weeks'}
+            path: /src/addon/web-ext-artifacts
+            type: directory
+      metadata:
+        name: Code Coverage Addon build
+        description: Build code-coverage webextension
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - taskId: {$eval: as_slugid("frontend_build")}
+      provisionerId: aws-provisioner-v1
+      workerType: github-worker
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 hour'}
+      payload:
+        maxRunTime: 3600
+        image: node
+        command:
+          - sh
+          - -lxce
+          - "git clone --quiet ${repository} /src && cd /src/frontend && git checkout ${head_rev} -b build &&
+            npm install --no-progress && npm run release"
+        artifacts:
+          public/frontend:
+            expires: {$fromNow: '2 weeks'}
+            path: /src/frontend/dist
+            type: directory
+      metadata:
+        name: Code Coverage Frontend build
+        description: Build web single page application
+        owner: [email protected]
+        source: https://github.com/mozilla/code-coverage
+
+    - $if: 'channel in ["testing", "production"]'
+      then:
+        taskId: {$eval: as_slugid("backend_deploy")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: aws-provisioner-v1
+        workerType: github-worker
+        dependencies:
+          - {$eval: as_slugid("backend_build")}
+        payload:
+          features:
+            taskclusterProxy: true
+          maxRunTime: 3600
+          image: "${taskboot_image}"
+          command:
+            - taskboot
+            - deploy-heroku
+            - --heroku-app
+            - "code-coverage-${channel}"
+            - web:public/code-coverage-backend.tar
+          env:
+            TASKCLUSTER_SECRET: "project/relman/code-coverage/deploy-${channel}"
+        scopes:
+          - "secrets:get:project/relman/code-coverage/deploy-${channel}"
+        metadata:
+          name: "Code Coverage Backend deployment (${channel})"
+          description: Deploy docker image on Heroku
+          owner: [email protected]
+          source: https://github.com/mozilla/code-coverage
+
+    - $if: 'channel in ["testing", "production"]'
+      then:
+        taskId: {$eval: as_slugid("bot_deploy")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: aws-provisioner-v1
+        workerType: github-worker
+        dependencies:
+          - {$eval: as_slugid("bot_build")}
+        payload:
+          features:
+            # Needed for access to secret
+            taskclusterProxy: true
+          maxRunTime: 3600
+          image: "${taskboot_image}"
+          env:
+            TASKCLUSTER_SECRET: "project/relman/code-coverage/deploy-${channel}"
+          command:
+            - taskboot
+            - push-artifact
+        scopes:
+          - "secrets:get:project/relman/code-coverage/deploy-${channel}"
+        metadata:
+          name: "Code Coverage Bot push (${channel})"
+          description: Push bot's docker image on repository
+          owner: [email protected]
+          source: https://github.com/mozilla/code-coverage
+
+    - $if: 'channel in ["testing", "production"]'
+      then:
+        taskId: {$eval: as_slugid("bot_hook")}
+        dependencies:
+          - {$eval: as_slugid("bot_deploy")}
+        scopes:
+          - "assume:hook-id:project-relman/code-coverage-${channel}"
+          - "hooks:modify-hook:project-relman/code-coverage-${channel}"
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '5 hours'}
+        provisionerId: aws-provisioner-v1
+        workerType: github-worker
+        payload:
+          features:
+            # Needed for access to hook api
+            taskclusterProxy: true
+          maxRunTime: 3600
+          image: "${taskboot_image}"
+          command:
+            - "/bin/sh"
+            - "-lcxe"
+            - "git clone --quiet ${repository} &&
+               cd code-coverage &&
+               git checkout ${head_rev} &&
+               sed -i -e 's/CHANNEL/${channel}/g' -e 's/REVISION/${head_rev}/g' bot/taskcluster-hook.json &&
+               taskboot --target . build-hook bot/taskcluster-hook.json project-relman code-coverage-${channel}"
+        metadata:
+          name: "Code Coverage Bot hook update (${channel})"
+          description: Update Taskcluster hook triggering the code-coverage tasks
+          owner: [email protected]
+          source: https://github.com/mozilla/code-coverage

backend/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3-slim
+
+ADD tools /src/tools
+ADD backend /src/backend
+
+RUN cd /src/tools && python setup.py install
+RUN cd /src/backend && python setup.py install
+
+CMD ["gunicorn", "code_coverage_backend.flask:app", "--timeout", "30"]

bot/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3-slim
+
+ADD tools /src/tools
+ADD bot /src/bot
+
+RUN /src/bot/ci/bootstrap.sh
+
+RUN cd /src/tools && python setup.py install
+RUN cd /src/bot && python setup.py install
+
+CMD ["code-coverage-bot"]

bot/ci/bootstrap.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -ex
+GRCOV_VERSION="v0.5.1"
+MERCURIAL_VERSION="4.8"
+
+apt-get update
+apt-get install --no-install-recommends -y mercurial=4.8.* curl lcov bzip2
+
+# Setup grcov
+curl -L https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-linux-x86_64.tar.bz2 | tar -C /usr/bin -xjv
+chmod +x /usr/bin/grcov
+
+# Setup mercurial with needed extensions
+hg clone https://hg.mozilla.org/hgcustom/version-control-tools /src/version-control-tools/
+ln -s /src/bot/ci/hgrc $HOME/.hgrc
+
+# Cleanup
+apt-get autoremove -y
+rm -rf /src/version-control-tools/.hg

bot/ci/hgrc

@@ -0,0 +1,4 @@
+[extensions]
+hgmo = /src/version-control-tools/hgext/hgmo
+pushlog = /src/version-control-tools/hgext/pushlog
+robustcheckout = /src/version-control-tools/hgext/robustcheckout