Step one for launching new PR integration testing (gatsbyjs#2920)

* Add lambda function for receiving github webhooks

* Record in graph.cool when new branches/commits are pushed

* Use node 6 syntax :-(...

* Add graphql operator

* Try try again

* missing the Bearer part of authorization

* Moer fixes

* Stringify author object — probably should change schema but this is a prototype so 🤷

* Ok, did need to break out data as JSON.stringify doesn't escape and don't want to double-stringify

* test commit

* Figuring out still Github's naming...

* Add Dockerfile + site build script

.gitignore

@@ -49,3 +49,4 @@ node_modules/
 .idea/
 .vscode/
 
+.serverless/

infrastructure/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:8
+RUN npm install -g [email protected]
+RUN yarn global add gatsby-cli gatsby-dev-cli
+RUN mkdir ~/.ssh && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+RUN git clone https://github.com/gatsbyjs/gatsby.git
+RUN wget https://github.com/github/hub/releases/download/v2.3.0-pre10/hub-linux-amd64-2.3.0-pre10.tgz --directory-prefix /usr/local/bin
+RUN cd gatsby && gatsby-dev --set-path-to-repo . && yarn bootstrap
+RUN mkdir -p gatsby/infrastructure
+COPY package.json gatsby/infrastructure/
+COPY yarn.lock gatsby/infrastructure/
+RUN cd gatsby/infrastructure && yarn
+COPY build-site.js gatsby/infrastructure/
+CMD ["node", "gatsby/infrastructure/build-site.js"]

infrastructure/build-site.js

@@ -0,0 +1,172 @@
+const shell = require("shelljs")
+const { spawn } = require("child_process")
+const path = require("path")
+const GraphQLClient = require("graphql-request").GraphQLClient
+const queue = require(`async/queue`)
+const s3 = require("s3")
+
+const commitId = process.env.CODEBUILD_SOURCE_VERSION
+
+const s3Client = s3.createClient({
+  s3Options: {
+    accessKeyId: process.env.accessKeyId,
+    secretAccessKey: process.env.secretAccessKey,
+    region: "us-east-1",
+    endpoint: "s3.us-east-1.amazonaws.com",
+  },
+})
+
+const client = new GraphQLClient(
+  "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr",
+  {
+    headers: {
+      Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`,
+    },
+  }
+)
+
+function createBuild(sha, commitObjId, url) {
+  return client.request(`
+    mutation {
+      createBuild(gitSha: "${sha}", result: BUILDING, url: "${url}", commitId: "${commitObjId}", logsIds: []) {
+        id
+      }
+    }
+  `)
+}
+
+function updateBuild(buildId, status) {
+  return client.request(`
+    mutation {
+      updateBuild(id: "${buildId}", result: ${status}) {
+        id
+      }
+    }
+  `)
+}
+
+function createLogLine(logLine, buildId, stderr = false) {
+  return client.request(
+    `
+    mutation createLogLine($text: String!, $buildId: ID!, $stderr: Boolean) {
+      createLogLine(text: $text, buildId: $buildId, stderr: $stderr) {
+        id
+      }
+    }
+  `,
+    { text: logLine, buildId, stderr }
+  )
+}
+
+const getCommitObjectByHash = commitId => {
+  return client
+    .request(
+      `
+      {
+        allCommits(
+          filter: { hash: "46f5ed9d8d05a9608bc9c98fcb5867a892149e94" }
+        ) {
+          id
+        }
+      }
+    `
+    )
+    .then(result => result.allCommits[0].id)
+}
+
+console.log(process.env)
+
+var q = queue(function({ logLine, buildId, stderr }, callback) {
+  createLogLine(logLine, buildId, stderr).then(callback)
+}, 1)
+
+const Main = async () => {
+  if (!process.env.PATH_TO_SITE) {
+    console.log("Missing required environment variable PATH_TO_SITE")
+    process.exit(1)
+  }
+
+  console.log(process.cwd())
+
+  // Navigate to Gatsby directory
+  shell.cd(`/gatsby/`)
+
+  // Ensure we have the latest code
+  shell.exec(`git fetch`)
+
+  // Check if the commit exists (later we'll grab pull requests as well
+  // but we're just building branches on gatsbyjs/gatsby while testing)
+  const commitExists = shell.exec(`git cat-file -e ${commitId}`)
+  console.log(`commitExists`, commitExists)
+  if (commitExists.code !== 0) {
+    process.exit(commitExists.code)
+  }
+
+  // It does! So let's checkout our commit and initialize the environment.
+  shell.exec(`git checkout -b newbranch ${commitId}`)
+  shell.exec(`yarn bootstrap`)
+
+  // Now go to the site and install
+  const pathToSite = path.join(`/gatsby/`, process.env.PATH_TO_SITE)
+  shell.cd(pathToSite)
+  shell.exec(`yarn`)
+  shell.exec(`gatsby-dev --scan-once --quiet`)
+
+  // Create the build
+  const commitObjId = await getCommitObjectByHash(commitId)
+  const url = `${process.env.PATH_TO_SITE}---${commitId}.gatsbydev.com`
+  const result = await createBuild(commitId, commitObjId, url)
+  const buildId = result.createBuild.id
+
+  // Start building!
+  const child = spawn(`gatsby`, [`build`])
+  child.on(`error`, err => console.log(`err:`, err))
+  child.stdout.pipe(process.stdout)
+  child.stderr.pipe(process.stderr)
+  child.stdout.on("data", data => {
+    // Create new logline
+    q.push({ logLine: data.toString("utf8"), buildId })
+  })
+  child.stderr.on("data", data => {
+    // Create new logline
+    q.push({ logLine: data.toString("utf8"), buildId, stderr: true })
+  })
+
+  // On end
+  child.on("exit", (code, signal) => {
+    console.log(
+      "child process exited with " + `code ${code} and signal ${signal}`
+    )
+    // Gatsby build failed
+    if (code !== 0) {
+      updateBuild(buildId, "FAILURE")
+      process.exit(code)
+    }
+    const publicDir = `${pathToSite}/public`
+    console.log(`uploading files from ${publicDir}`)
+
+    // 1. Push built files to s3 bucket
+    const upload = s3Client.uploadDir({
+      localDir: publicDir,
+      s3Params: {
+        Prefix: url,
+        Bucket: `gatsby-js-builds`,
+      },
+    })
+    upload.on(`fileUploadEnd`, () =>
+      console.log(`${upload.progressAmount / upload.progressTotal}`)
+    )
+    upload.on(`error`, error => {
+      console.error(error)
+      updateBuild(buildId, "FAILURE").then(() => {
+        process.exit(code)
+      })
+    })
+    // 2. Write final status of build and exit.
+    upload.on(`end`, () => {
+      updateBuild(buildId, "SUCCESS").then(() => process.exit())
+    })
+  })
+}
+
+Main()

infrastructure/functions/onGithubWebhook/handler.js

@@ -0,0 +1,104 @@
+// handler.js
+
+"use strict"
+const axios = require("axios")
+const Libhoney = require("libhoney").default
+const flatten = require("flat")
+const GraphQLClient = require("graphql-request").GraphQLClient
+
+const hny = new Libhoney({
+  writeKey: process.env.HONEYCOMB_KEY,
+  dataset: "gatsbyjs-os.lambda.github-webhook",
+})
+
+console.log(`env vars`, process.env)
+
+const client = new GraphQLClient(
+  "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr",
+  {
+    headers: {
+      Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`,
+    },
+  }
+)
+
+function createBranch(branch) {
+  hny.sendNow({ createBranch: branch })
+  return client.request(`
+    mutation {
+      createBranch(name: "${branch}", commits: []) {
+        id
+      }
+    }
+  `)
+}
+
+function getBranch(branch) {
+  return client.request(`
+    query {
+      allBranches(filter: {name: "${branch}"}) {
+        id
+        name
+      }
+    }
+  `)
+}
+
+const createBranchIfDoesNotExist = branch =>
+  getBranch(branch).then(result => {
+    if (result.allBranches.length === 0) {
+      return createBranch(branch).then(newBranch => {
+        console.log(`newBranch`, newBranch)
+        return newBranch.createBranch.id
+      })
+    } else {
+      console.log(`allBranches`, result)
+      return result.allBranches[0].id
+    }
+  })
+
+const createCommit = (commit, branchId) => {
+  hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit))
+  return client.request(`
+    mutation {
+      createCommit(authorName: "${commit.author.name}",authorUsername: "${commit
+    .author.username}", authorEmail: "${commit.author
+    .email}", hash: "${commit.tree_id}", message: "${commit.message}", branchIds: ["${branchId}"]) {
+        id
+      }
+    }
+  `)
+}
+
+module.exports.event = function(event, context, callback) {
+  if (event && event.body) {
+    event.body = JSON.parse(event.body)
+  }
+  console.log(event) // Contains incoming request data (e.g., query params, headers and more)
+  hny.sendNow(flatten(event))
+
+  const response = {
+    statusCode: 200,
+    body: ``,
+  }
+
+  new Promise(resolve => {
+    // Commit push
+    if (
+      event.headers[`X-GitHub-Event`] === `push` &&
+      event.body.ref.split(`/`)[1] === `heads`
+    ) {
+      const branchName = event.body.ref.split(`/`)[2]
+
+      // Check if the branch exists
+      createBranchIfDoesNotExist(branchName).then(branchId => {
+        // Create commits
+        resolve(
+          Promise.all(
+            event.body.commits.map(commit => createCommit(commit, branchId))
+          )
+        )
+      })
+    }
+  }).then(() => callback(null, response))
+}

infrastructure/functions/onGithubWebhook/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "ongithubwebhook",
+  "version": "1.0.0",
+  "description": "",
+  "main": "handler.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "axios": "^0.17.0",
+    "flat": "^4.0.0",
+    "graphql-request": "^1.4.0",
+    "libhoney": "^1.0.0-beta.10"
+  }
+}