From a63c8d7ee65034decad01b0fbc7e35ff9e741573 Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Wed, 6 Mar 2024 18:41:26 -0500
Subject: [PATCH 1/9] EPD-621 Updates for CI

---
 .github/workflows/ci.yml              |   8 +-
 .github/workflows/e2e.yml             |  80 ++++++++++++
 .prettierignore                       |   1 +
 e2e/python/requirements.txt           |   1 +
 e2e/python/run.py                     |  80 ++++++++++++
 e2e/typescript/package.json           |   9 ++
 e2e/typescript/run.ts                 |  88 ++++++++++++++
 e2e/typescript/tsconfig.json          |   8 ++
 package-lock.json                     |   8 +-
 package.json                          |   2 +-
 src/handlers/testing/exec/emitter.ts  |   2 +-
 src/handlers/testing/exec/index.ts    | 163 ++++++++++++++++---------
 src/handlers/testing/exec/util/ci.ts  | 168 ++++++++++++++++++++++++++
 src/handlers/testing/exec/util/net.ts |  35 ++++++
 14 files changed, 585 insertions(+), 68 deletions(-)
 create mode 100644 .github/workflows/e2e.yml
 create mode 100644 e2e/python/requirements.txt
 create mode 100644 e2e/python/run.py
 create mode 100644 e2e/typescript/package.json
 create mode 100644 e2e/typescript/run.ts
 create mode 100644 e2e/typescript/tsconfig.json
 create mode 100644 src/handlers/testing/exec/util/ci.ts
 create mode 100644 src/handlers/testing/exec/util/net.ts

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 235fe1a..7f4dba6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,6 +8,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+  TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
+
 jobs:
   ci:
     runs-on: ubuntu-latest
@@ -46,10 +50,6 @@ jobs:
 
       - name: Run build
         run: npm run build
-        env:
-          TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
 
       - name: Run testing exec
         run: npx autoblocks testing exec -- echo "hi"
-        env:
-          AUTOBLOCKS_API_KEY: test
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
new file mode 100644
index 0000000..59d0889
--- /dev/null
+++ b/.github/workflows/e2e.yml
@@ -0,0 +1,80 @@
+name: E2E
+
+on:
+  # Run on different types of events to ensure we are
+  # handling the git data correctly in each scenario
+  push:
+  pull_request:
+  schedule:
+    - cron: '17 15 * * *'
+
+env:
+  AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+  TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
+
+jobs:
+  py:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # For debugging purposes
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install CLI dependencies
+        run: npm ci
+
+      - name: Build CLI
+        run: npm run build
+
+      - name: Install dependencies in e2e/python
+        run: pip install -r requirements.txt
+        working-directory: e2e/python
+
+      - name: Run tests in e2e/python
+        run: ../../bin/cli.js testing exec -- python3 run.py
+        working-directory: e2e/python
+        env:
+          PYTHONPATH: ${{ github.workspace }}/e2e/python
+
+  ts:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # For debugging purposes
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install CLI dependencies
+        run: npm ci
+
+      - name: Build CLI
+        run: npm run build
+
+      - name: Install dependencies in e2e/typescript
+        run: npm install
+        working-directory: e2e/typescript
+
+      - name: Run tests in e2e/typescript
+        run: ../../bin/cli.js testing exec -- npx tsx run.ts
+        working-directory: e2e/typescript
diff --git a/.prettierignore b/.prettierignore
index 30748ad..03b4f5f 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -2,3 +2,4 @@
 *.py
 CODEOWNERS
 *.sh
+*.txt
diff --git a/e2e/python/requirements.txt b/e2e/python/requirements.txt
new file mode 100644
index 0000000..316d144
--- /dev/null
+++ b/e2e/python/requirements.txt
@@ -0,0 +1 @@
+autoblocksai
diff --git a/e2e/python/run.py b/e2e/python/run.py
new file mode 100644
index 0000000..24959f9
--- /dev/null
+++ b/e2e/python/run.py
@@ -0,0 +1,80 @@
+import uuid
+import random
+import asyncio
+import dataclasses
+
+from autoblocks.testing.models import BaseTestCase
+from autoblocks.testing.models import BaseTestEvaluator
+from autoblocks.testing.models import Evaluation
+from autoblocks.testing.models import Threshold
+from autoblocks.testing.util import md5
+from autoblocks.testing.run import run_test_suite
+
+
+@dataclasses.dataclass
+class MyTestCase(BaseTestCase):
+    input: str
+    expected_substrings: list[str]
+
+    def hash(self) -> str:
+        return md5(self.input)
+    
+
+async def test_fn(test_case: MyTestCase) -> str:
+    await asyncio.sleep(random.random())
+
+    substrings = test_case.input.split("-")
+    if random.random() < 0.2:
+        substrings.pop()
+
+    return "-".join(substrings)
+
+
+class HasAllSubstrings(BaseTestEvaluator):
+    id = "has-all-substrings"
+
+    def evaluate_test_case(self, test_case: MyTestCase, output: str) -> Evaluation:
+        score = 1 if all(s in output for s in test_case.expected_substrings) else 0
+        return Evaluation(
+            score=score,
+            threshold=Threshold(gte=1),
+        )
+
+
+class IsFriendly(BaseTestEvaluator):
+    id = "is-friendly"
+
+    async def get_score(self, output: str) -> float:
+        await asyncio.sleep(random.random())
+        return random.random()
+
+    async def evaluate_test_case(self, test_case: BaseTestCase, output: str) -> Evaluation:
+        score = await self.get_score(output)
+        return Evaluation(
+            score=score,
+        )
+    
+
+def gen_test_cases(n: int) -> list[MyTestCase]:
+    test_cases = []
+    for _ in range(n):
+        random_id = str(uuid.uuid4())
+        test_cases.append(
+            MyTestCase(
+                input=random_id,
+                expected_substrings=random_id.split("-"),
+            ),
+        )
+    return test_cases
+
+
+if __name__ == "__main__":
+  run_test_suite(
+      id="my-test-suite",
+      fn=test_fn,
+      test_cases=gen_test_cases(40),
+      evaluators=[
+          HasAllSubstrings(),
+          IsFriendly(),
+      ],
+  )
diff --git a/e2e/typescript/package.json b/e2e/typescript/package.json
new file mode 100644
index 0000000..db8ef82
--- /dev/null
+++ b/e2e/typescript/package.json
@@ -0,0 +1,9 @@
+{
+  "name": "ts-e2e",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "@autoblocks/client": "*",
+    "typescript": "*"
+  }
+}
diff --git a/e2e/typescript/run.ts b/e2e/typescript/run.ts
new file mode 100644
index 0000000..ed10494
--- /dev/null
+++ b/e2e/typescript/run.ts
@@ -0,0 +1,88 @@
+import {
+  BaseTestEvaluator,
+  runTestSuite,
+  type Evaluation,
+} from '@autoblocks/client/testing';
+import * as crypto from 'crypto';
+
+interface MyTestCase {
+  input: string;
+  expectedSubstrings: string[];
+}
+
+async function testFn({ testCase }: { testCase: MyTestCase }): Promise<string> {
+  // Simulate doing work
+  await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
+
+  const substrings = testCase.input.split('-');
+  if (Math.random() < 0.2) {
+    // Remove a substring randomly. This will cause about 20% of the test cases to fail
+    // the "has-all-substrings" evaluator.
+    substrings.pop();
+  }
+
+  return substrings.join('-');
+}
+
+class HasAllSubstrings extends BaseTestEvaluator<MyTestCase, string> {
+  id = 'has-all-substrings';
+
+  evaluateTestCase(args: { testCase: MyTestCase; output: string }): Evaluation {
+    const score = args.testCase.expectedSubstrings.every((s) =>
+      args.output.includes(s),
+    )
+      ? 1
+      : 0;
+
+    return {
+      score,
+      threshold: {
+        gte: 1,
+      },
+    };
+  }
+}
+
+class IsFriendly extends BaseTestEvaluator<MyTestCase, string> {
+  id = 'is-friendly';
+
+  async getScore(output: string): Promise<number> {
+    // eslint-disable-next-line no-console
+    console.log(`Determining score for output: ${output}`);
+    await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
+    return Math.random();
+  }
+
+  async evaluateTestCase(args: {
+    testCase: MyTestCase;
+    output: string;
+  }): Promise<Evaluation> {
+    const score = await this.getScore(args.output);
+
+    return {
+      score,
+    };
+  }
+}
+
+function genTestCases(n: number): MyTestCase[] {
+  const testCases: MyTestCase[] = [];
+  for (let i = 0; i < n; i++) {
+    const randomId = crypto.randomUUID();
+    testCases.push({
+      input: randomId,
+      expectedSubstrings: randomId.split('-'),
+    });
+  }
+  return testCases;
+}
+
+(async () => {
+  await runTestSuite<MyTestCase, string>({
+    id: 'my-test-suite',
+    fn: testFn,
+    testCaseHash: ['input'],
+    testCases: genTestCases(40),
+    evaluators: [new HasAllSubstrings(), new IsFriendly()],
+  });
+})();
diff --git a/e2e/typescript/tsconfig.json b/e2e/typescript/tsconfig.json
new file mode 100644
index 0000000..6454baa
--- /dev/null
+++ b/e2e/typescript/tsconfig.json
@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "moduleResolution": "node",
+    "module": "esnext",
+    "target": "esnext"
+  },
+  "include": ["**/*.ts"]
+}
diff --git a/package-lock.json b/package-lock.json
index 595c92c..23ba828 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,7 +10,7 @@
       "license": "MIT",
       "dependencies": {
         "@actions/exec": "^1.1.1",
-        "@autoblocks/client": "^0.0.32",
+        "@autoblocks/client": "^0.0.33",
         "@hono/node-server": "^1.7.0",
         "@hono/zod-validator": "^0.1.11",
         "hono": "^3.12.12",
@@ -96,9 +96,9 @@
       }
     },
     "node_modules/@autoblocks/client": {
-      "version": "0.0.32",
-      "resolved": "https://registry.npmjs.org/@autoblocks/client/-/client-0.0.32.tgz",
-      "integrity": "sha512-XYd2xAqsEUUxGHqILNJXFBrkSXiG7T3UTIcc/rgQjJaVfYIl0Mb3Z/em5l4IDUg44JLjqXoxc73vlu/JBDcUFw==",
+      "version": "0.0.33",
+      "resolved": "https://registry.npmjs.org/@autoblocks/client/-/client-0.0.33.tgz",
+      "integrity": "sha512-uivJ5XiFAWlK80n7IRTwRoY5oeIHIyy/9KbKCU8aLt4o2s/A0bgYsAgfqIjhJ2/ahiStwapAFnnt1tBfyibPpQ==",
       "dependencies": {
         "zod": "^3.21.4"
       },
diff --git a/package.json b/package.json
index 7c5e572..3c7a04f 100644
--- a/package.json
+++ b/package.json
@@ -30,7 +30,7 @@
   "homepage": "https://github.com/autoblocksai/cli#readme",
   "dependencies": {
     "@actions/exec": "^1.1.1",
-    "@autoblocks/client": "^0.0.32",
+    "@autoblocks/client": "^0.0.33",
     "@hono/node-server": "^1.7.0",
     "@hono/zod-validator": "^0.1.11",
     "hono": "^3.12.12",
diff --git a/src/handlers/testing/exec/emitter.ts b/src/handlers/testing/exec/emitter.ts
index 9c8e76f..4aab3c4 100644
--- a/src/handlers/testing/exec/emitter.ts
+++ b/src/handlers/testing/exec/emitter.ts
@@ -12,7 +12,7 @@ export enum EventName {
 
 // Zod schemas for event data
 const zConsoleLogSchema = z.object({
-  ctx: z.enum(['cmd', 'cli', 'cli-server']),
+  ctx: z.enum(['cmd', 'cli']),
   level: z.enum(['debug', 'info', 'warn', 'error']),
   message: z.string(),
 });
diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts
index 2d0eb2d..4196a63 100644
--- a/src/handlers/testing/exec/index.ts
+++ b/src/handlers/testing/exec/index.ts
@@ -3,10 +3,12 @@ import { serve } from '@hono/node-server';
 import type { ServerType } from '@hono/node-server/dist/types';
 import { zValidator } from '@hono/zod-validator';
 import { Hono, type Context } from 'hono';
-import net from 'net';
+
 import { z, type SafeParseReturnType } from 'zod';
 import { renderTestProgress } from './components/progress';
 import { EventName, emitter, type EventSchemas } from './emitter';
+import { makeCIContext, type CIContext } from './util/ci';
+import { findAvailablePort } from './util/net';
 
 type UncaughtError = EventSchemas[EventName.UNCAUGHT_ERROR];
 
@@ -21,37 +23,6 @@ interface TestCaseEvent {
   };
 }
 
-async function findAvailablePort(args: { startPort: number }): Promise<number> {
-  return new Promise((resolve, reject) => {
-    const tryListening = (port: number) => {
-      const server = net.createServer();
-
-      server.listen(port, () => {
-        server.once('close', () => {
-          resolve(port);
-        });
-        server.close();
-      });
-
-      server.on('error', (err) => {
-        if ((err as { code?: string } | undefined)?.code === 'EADDRINUSE') {
-          const nextPort = port + 1;
-          emitter.emit(EventName.CONSOLE_LOG, {
-            ctx: 'cli-server',
-            level: 'info',
-            message: `Port ${port} is in use, trying port ${nextPort}...`,
-          });
-          tryListening(nextPort);
-        } else {
-          reject(err);
-        }
-      });
-    };
-
-    tryListening(args.startPort);
-  });
-}
-
 /**
  * Manages the state of the current run
  */
@@ -90,6 +61,18 @@ class RunManager {
    */
   hasAnyFailedEvaluations: boolean;
 
+  /**
+   * The CI context, parsed from environment variables and local git data.
+   */
+  private ciContext: CIContext | undefined;
+
+  /**
+   * Only relevant for CI runs. This is returned from our REST API
+   * when we POST to /builds at the beginning of a build on CI.
+   * It's used in subsequent requests to associate runs with the build.
+   */
+  private ciBuildId: string | undefined;
+
   constructor(args: { apiKey: string; runMessage: string | undefined }) {
     this.apiKey = args.apiKey;
     this.message = args.runMessage;
@@ -102,7 +85,8 @@ class RunManager {
   }
 
   private async post<T>(path: string, body?: unknown): Promise<T> {
-    const resp = await fetch(`https://api.autoblocks.ai${path}`, {
+    const subpath = this.isCI ? '/testing/ci' : '/testing/local';
+    const resp = await fetch(`https://api.autoblocks.ai${subpath}${path}`, {
       method: 'POST',
       body: body ? JSON.stringify(body) : undefined,
       headers: {
@@ -118,13 +102,75 @@ class RunManager {
     return resp.json();
   }
 
+  private get isCI(): boolean {
+    return process.env.CI === 'true';
+  }
+
+  async makeCIContext(): Promise<CIContext | undefined> {
+    if (!this.isCI) {
+      return undefined;
+    }
+    if (!this.ciContext) {
+      try {
+        this.ciContext = await makeCIContext();
+      } catch (err) {
+        emitter.emit(EventName.CONSOLE_LOG, {
+          ctx: 'cli',
+          level: 'error',
+          message: `Failed to get CI context: ${err}`,
+        });
+        throw err;
+      }
+    }
+    return this.ciContext;
+  }
+
+  async setup(): Promise<void> {
+    const ciContext = await this.makeCIContext();
+
+    if (!ciContext) {
+      return;
+    }
+
+    emitter.emit(EventName.CONSOLE_LOG, {
+      ctx: 'cli',
+      level: 'debug',
+      message: `Running in CI environment: ${JSON.stringify(ciContext, null, 2)}`,
+    });
+
+    const { id } = await this.post<{ id: string }>('/builds', {
+      gitProvider: ciContext.gitProvider,
+      repositoryExternalId: ciContext.repoId,
+      repositoryName: ciContext.repoName,
+      repositoryHtmlUrl: ciContext.repoHtmlUrl,
+      // TODO: figure out how to get internal branch ID from github
+      branchExternalId: ciContext.branchName,
+      branchName: ciContext.branchName,
+      isDefaultBranch: ciContext.branchName === ciContext.defaultBranchName,
+      ciProvider: ciContext.ciProvider,
+      buildHtmlUrl: ciContext.buildHtmlUrl,
+      commitSha: ciContext.commitSha,
+      commitMessage: ciContext.commitMessage,
+      commitComitterName: ciContext.commitCommitterName,
+      commitCommitterEmail: ciContext.commitCommitterEmail,
+      commitAuthorName: ciContext.commitAuthorName,
+      commitAuthorEmail: ciContext.commitAuthorEmail,
+      commitCommittedDate: ciContext.commitCommittedDate,
+      pullRequestNumber: ciContext.pullRequestNumber,
+      pullRequestTitle: ciContext.pullRequestTitle,
+    });
+
+    this.ciBuildId = id;
+  }
+
   async handleStartRun(args: { testExternalId: string }): Promise<string> {
     emitter.emit(EventName.RUN_STARTED, {
       testExternalId: args.testExternalId,
     });
-    const { id } = await this.post<{ id: string }>('/testing/local/runs', {
+    const { id } = await this.post<{ id: string }>('/runs', {
       testExternalId: args.testExternalId,
       message: this.message,
+      buildId: this.ciBuildId,
     });
     this.testExternalIdToRunId[args.testExternalId] = id;
     return id;
@@ -132,9 +178,13 @@ class RunManager {
 
   async handleEndRun(args: { testExternalId: string }): Promise<void> {
     emitter.emit(EventName.RUN_ENDED, { testExternalId: args.testExternalId });
-    const runId = this.currentRunId({ testExternalId: args.testExternalId });
-    await this.post(`/testing/local/runs/${runId}/end`);
-    delete this.testExternalIdToRunId[args.testExternalId];
+
+    try {
+      const runId = this.currentRunId({ testExternalId: args.testExternalId });
+      await this.post(`/runs/${runId}/end`);
+    } finally {
+      delete this.testExternalIdToRunId[args.testExternalId];
+    }
   }
 
   async endAllRuns() {
@@ -203,7 +253,7 @@ class RunManager {
       testExternalId: args.testExternalId,
     });
     const { id: resultId } = await this.post<{ id: string }>(
-      `/testing/local/runs/${runId}/results`,
+      `/runs/${runId}/results`,
       {
         testCaseHash: args.testCaseHash,
         testCaseBody: args.testCaseBody,
@@ -284,16 +334,13 @@ class RunManager {
       testExternalId: args.testExternalId,
     });
 
-    await this.post(
-      `/testing/local/runs/${runId}/results/${testCaseResultId}/evaluations`,
-      {
-        evaluatorExternalId: args.evaluatorExternalId,
-        score: args.score,
-        passed,
-        threshold: args.threshold,
-        metadata: args.metadata,
-      },
-    );
+    await this.post(`/runs/${runId}/results/${testCaseResultId}/evaluations`, {
+      evaluatorExternalId: args.evaluatorExternalId,
+      score: args.score,
+      passed,
+      threshold: args.threshold,
+      metadata: args.metadata,
+    });
 
     if (passed === false) {
       this.hasAnyFailedEvaluations = true;
@@ -309,7 +356,7 @@ function createHonoApp(runManager: RunManager): Hono {
 
   app.onError((err, c) => {
     emitter.emit(EventName.CONSOLE_LOG, {
-      ctx: 'cli-server',
+      ctx: 'cli',
       level: 'error',
       message: `${c.req.method} ${c.req.path}: ${err.message}`,
     });
@@ -322,7 +369,7 @@ function createHonoApp(runManager: RunManager): Hono {
   ) => {
     if (!result.success) {
       emitter.emit(EventName.CONSOLE_LOG, {
-        ctx: 'cli-server',
+        ctx: 'cli',
         level: 'error',
         message: `${c.req.method} ${c.req.path}: ${result.error}`,
       });
@@ -508,6 +555,8 @@ export async function exec(args: {
     runMessage: args.runMessage,
   });
 
+  await runManager.setup();
+
   let runningServer: ServerType | undefined = undefined;
 
   const app = createHonoApp(runManager);
@@ -540,17 +589,11 @@ export async function exec(args: {
       const serverAddress = `http://localhost:${addressInfo.port}`;
 
       emitter.emit(EventName.CONSOLE_LOG, {
-        ctx: 'cli-server',
+        ctx: 'cli',
         level: 'info',
         message: `Listening on ${serverAddress}`,
       });
 
-      // Set environment variables for the SDKs to use
-      const env = {
-        ...process.env,
-        AUTOBLOCKS_CLI_SERVER_ADDRESS: serverAddress,
-      };
-
       emitter.emit(EventName.CONSOLE_LOG, {
         ctx: 'cli',
         level: 'info',
@@ -559,7 +602,11 @@ export async function exec(args: {
 
       // Execute the command
       execCommand(args.command, args.commandArgs, {
-        env,
+        // Set environment variables for the SDKs to use
+        env: {
+          ...process.env,
+          AUTOBLOCKS_CLI_SERVER_ADDRESS: serverAddress,
+        },
         // will return error code as response (even if it's non-zero)
         ignoreReturnCode: true,
         silent: true,
diff --git a/src/handlers/testing/exec/util/ci.ts b/src/handlers/testing/exec/util/ci.ts
new file mode 100644
index 0000000..c2a2d81
--- /dev/null
+++ b/src/handlers/testing/exec/util/ci.ts
@@ -0,0 +1,168 @@
+import { z } from 'zod';
+import { getExecOutput } from '@actions/exec';
+import fs from 'fs/promises';
+
+export interface CIContext {
+  gitProvider: 'github';
+  repoId: string;
+  repoName: string;
+  repoHtmlUrl: string;
+  branchName: string;
+  defaultBranchName: string;
+  ciProvider: 'github';
+  buildHtmlUrl: string;
+  commitSha: string;
+  commitMessage: string;
+  commitCommitterName: string;
+  commitCommitterEmail: string;
+  commitAuthorName: string;
+  commitAuthorEmail: string;
+  commitCommittedDate: string;
+  pullRequestNumber: number | null;
+  pullRequestTitle: string | null;
+}
+
+const zGitHubEnvSchema = z.object({
+  GITHUB_ACTIONS: z.literal('true'),
+  GITHUB_SERVER_URL: z.string(),
+  GITHUB_REPOSITORY: z.string(),
+  GITHUB_REF_NAME: z.string(),
+  GITHUB_EVENT_PATH: z.string(),
+  GITHUB_SHA: z.string(),
+  GITHUB_RUN_ID: z.string(),
+  GITHUB_RUN_ATTEMPT: z.string(),
+});
+
+const zCommitSchema = z.object({
+  sha: z.string(),
+  message: z.string(),
+  authorName: z.string(),
+  authorEmail: z.string(),
+  committerName: z.string(),
+  committerEmail: z.string(),
+  committedDate: z.string(),
+});
+
+const zPullRequestSchema = z.object({
+  number: z.number(),
+  title: z.string(),
+  head: z.object({
+    ref: z.string(),
+  }),
+});
+
+const zRepositorySchema = z.object({
+  id: z.coerce.string(),
+  default_branch: z.string(),
+});
+
+type Commit = z.infer<typeof zCommitSchema>;
+type PullRequest = z.infer<typeof zPullRequestSchema>;
+type Repository = z.infer<typeof zRepositorySchema>;
+
+function pullRequestFromEvent(event: unknown): PullRequest | null {
+  const parsed = zPullRequestSchema.safeParse(event);
+  if (parsed.success) {
+    return parsed.data;
+  }
+  return null;
+}
+
+function repositoryFromEvent(event: unknown): Repository {
+  return zRepositorySchema.parse(event);
+}
+
+async function parseCommitFromGitLog(sha: string): Promise<Commit> {
+  const commitMessageKey = 'message';
+
+  const logFormat = [
+    'sha=%H',
+    'authorName=%an',
+    'authorEmail=%ae',
+    'committerName=%cn',
+    'committerEmail=%ce',
+    'committedDate=%aI',
+    // This should be last because it can contain multiple lines
+    `${commitMessageKey}=%B`,
+  ].join('%n');
+
+  const { stdout } = await getExecOutput(
+    'git',
+    ['show', sha, '--quiet', `--format=${logFormat}`],
+    {
+      silent: true,
+    },
+  );
+  const lines = stdout.split('\n');
+
+  const data: Record<string, string> = {};
+
+  while (lines.length) {
+    const line = lines.shift();
+    if (!line) {
+      break;
+    }
+
+    // Split on the first =
+    const idx = line.indexOf('=');
+    const [key, value] = [line.slice(0, idx), line.slice(idx + 1)];
+
+    if (key === commitMessageKey) {
+      // Once we've reached the commit message key, the remaining lines are the commit message.
+      // We only keep the first line of the commit message, though, since some commit
+      // messages can be very long.
+      data[commitMessageKey] = value;
+      break;
+    }
+
+    data[key] = value;
+  }
+
+  return zCommitSchema.parse(data);
+}
+
+export async function makeCIContext(): Promise<CIContext> {
+  const env = zGitHubEnvSchema.parse(process.env);
+  const commit = await parseCommitFromGitLog(env.GITHUB_SHA);
+
+  // GitHub Actions are triggered by webhook events, and the event payload is
+  // stored in a JSON file at $GITHUB_EVENT_PATH.
+  // You can see the schema of the various webhook payloads at:
+  // https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads
+  const event = JSON.parse(await fs.readFile(env.GITHUB_EVENT_PATH, 'utf-8'));
+
+  const repository = repositoryFromEvent(event.repository);
+  const pullRequest = pullRequestFromEvent(event.pull_request);
+
+  return {
+    gitProvider: 'github',
+    repoId: repository.id,
+    // full repo name including owner, e.g. 'octocat/Hello-World'
+    repoName: env.GITHUB_REPOSITORY,
+    repoHtmlUrl: [env.GITHUB_SERVER_URL, env.GITHUB_REPOSITORY].join('/'),
+    // When it's a `push` event, the branch name is in `GITHUB_REF_NAME`, but on the `pull_request`
+    // event we want to use event.pull_request.head.ref, since `GITHUB_REF_NAME` will contain the
+    // name of the merge commit for the PR, like 5/merge.
+    branchName: pullRequest?.head.ref || env.GITHUB_REF_NAME,
+    defaultBranchName: repository.default_branch,
+    ciProvider: 'github',
+    buildHtmlUrl: [
+      env.GITHUB_SERVER_URL,
+      env.GITHUB_REPOSITORY,
+      'actions',
+      'runs',
+      env.GITHUB_RUN_ID,
+      'attempts',
+      env.GITHUB_RUN_ATTEMPT,
+    ].join('/'),
+    commitSha: commit.sha,
+    commitMessage: commit.message,
+    commitCommitterName: commit.committerName,
+    commitCommitterEmail: commit.committerEmail,
+    commitAuthorName: commit.authorName,
+    commitAuthorEmail: commit.authorEmail,
+    commitCommittedDate: commit.committedDate,
+    pullRequestNumber: pullRequest?.number ?? null,
+    pullRequestTitle: pullRequest?.title || null,
+  };
+}
diff --git a/src/handlers/testing/exec/util/net.ts b/src/handlers/testing/exec/util/net.ts
new file mode 100644
index 0000000..3112cda
--- /dev/null
+++ b/src/handlers/testing/exec/util/net.ts
@@ -0,0 +1,35 @@
+import net from 'net';
+import { emitter, EventName } from '../emitter';
+
+export async function findAvailablePort(args: {
+  startPort: number;
+}): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const tryListening = (port: number) => {
+      const server = net.createServer();
+
+      server.listen(port, () => {
+        server.once('close', () => {
+          resolve(port);
+        });
+        server.close();
+      });
+
+      server.on('error', (err) => {
+        if ((err as { code?: string } | undefined)?.code === 'EADDRINUSE') {
+          const nextPort = port + 1;
+          emitter.emit(EventName.CONSOLE_LOG, {
+            ctx: 'cli',
+            level: 'info',
+            message: `Port ${port} is in use, trying port ${nextPort}...`,
+          });
+          tryListening(nextPort);
+        } else {
+          reject(err);
+        }
+      });
+    };
+
+    tryListening(args.startPort);
+  });
+}

From 20e50962e3dee62a5e16f91e00f77b6c529c5618 Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Thu, 7 Mar 2024 16:31:03 -0500
Subject: [PATCH 2/9] branchId?

---
 .github/workflows/ci.yml             |   1 +
 .github/workflows/e2e.yml            |   1 +
 package-lock.json                    | 185 ++++++++++++++++++++++++++-
 package.json                         |   1 +
 src/handlers/testing/exec/index.ts   |   3 +-
 src/handlers/testing/exec/util/ci.ts |  21 ++-
 6 files changed, 203 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7f4dba6..db502fd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,6 +10,7 @@ concurrency:
 
 env:
   AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
 
 jobs:
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 59d0889..17924d9 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -10,6 +10,7 @@ on:
 
 env:
   AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
 
 jobs:
diff --git a/package-lock.json b/package-lock.json
index 23ba828..2ac5e52 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,6 +10,7 @@
       "license": "MIT",
       "dependencies": {
         "@actions/exec": "^1.1.1",
+        "@actions/github": "^6.0.0",
         "@autoblocks/client": "^0.0.33",
         "@hono/node-server": "^1.7.0",
         "@hono/zod-validator": "^0.1.11",
@@ -56,6 +57,26 @@
         "@actions/io": "^1.0.1"
       }
     },
+    "node_modules/@actions/github": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz",
+      "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==",
+      "dependencies": {
+        "@actions/http-client": "^2.2.0",
+        "@octokit/core": "^5.0.1",
+        "@octokit/plugin-paginate-rest": "^9.0.0",
+        "@octokit/plugin-rest-endpoint-methods": "^10.0.0"
+      }
+    },
+    "node_modules/@actions/http-client": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.1.tgz",
+      "integrity": "sha512-KhC/cZsq7f8I4LfZSJKgCvEwfkE8o1538VoBeoGzokVLLnbFDEAdFD3UhoMklxo2un9NJVBdANOresx7vTHlHw==",
+      "dependencies": {
+        "tunnel": "^0.0.6",
+        "undici": "^5.25.4"
+      }
+    },
     "node_modules/@actions/io": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
@@ -555,6 +576,14 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
+      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@hono/node-server": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.7.0.tgz",
@@ -806,6 +835,124 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@octokit/auth-token": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz",
+      "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==",
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/core": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz",
+      "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==",
+      "dependencies": {
+        "@octokit/auth-token": "^4.0.0",
+        "@octokit/graphql": "^7.0.0",
+        "@octokit/request": "^8.0.2",
+        "@octokit/request-error": "^5.0.0",
+        "@octokit/types": "^12.0.0",
+        "before-after-hook": "^2.2.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/endpoint": {
+      "version": "9.0.4",
+      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.4.tgz",
+      "integrity": "sha512-DWPLtr1Kz3tv8L0UvXTDP1fNwM0S+z6EJpRcvH66orY6Eld4XBMCSYsaWp4xIm61jTWxK68BrR7ibO+vSDnZqw==",
+      "dependencies": {
+        "@octokit/types": "^12.0.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/graphql": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz",
+      "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==",
+      "dependencies": {
+        "@octokit/request": "^8.0.1",
+        "@octokit/types": "^12.0.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/openapi-types": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+      "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="
+    },
+    "node_modules/@octokit/plugin-paginate-rest": {
+      "version": "9.2.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.1.tgz",
+      "integrity": "sha512-wfGhE/TAkXZRLjksFXuDZdmGnJQHvtU/joFQdweXUgzo1XwvBCD4o4+75NtFfjfLK5IwLf9vHTfSiU3sLRYpRw==",
+      "dependencies": {
+        "@octokit/types": "^12.6.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@octokit/core": "5"
+      }
+    },
+    "node_modules/@octokit/plugin-rest-endpoint-methods": {
+      "version": "10.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz",
+      "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==",
+      "dependencies": {
+        "@octokit/types": "^12.6.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@octokit/core": "5"
+      }
+    },
+    "node_modules/@octokit/request": {
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.2.0.tgz",
+      "integrity": "sha512-exPif6x5uwLqv1N1irkLG1zZNJkOtj8bZxuVHd71U5Ftuxf2wGNvAJyNBcPbPC+EBzwYEbBDdSFb8EPcjpYxPQ==",
+      "dependencies": {
+        "@octokit/endpoint": "^9.0.0",
+        "@octokit/request-error": "^5.0.0",
+        "@octokit/types": "^12.0.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/request-error": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz",
+      "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==",
+      "dependencies": {
+        "@octokit/types": "^12.0.0",
+        "deprecation": "^2.0.0",
+        "once": "^1.4.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/types": {
+      "version": "12.6.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+      "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+      "dependencies": {
+        "@octokit/openapi-types": "^20.0.0"
+      }
+    },
     "node_modules/@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -1382,6 +1529,11 @@
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "dev": true
     },
+    "node_modules/before-after-hook": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
+      "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ=="
+    },
     "node_modules/binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -1708,6 +1860,11 @@
       "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
       "dev": true
     },
+    "node_modules/deprecation": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
+      "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ=="
+    },
     "node_modules/dir-glob": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@@ -3326,7 +3483,6 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-      "dev": true,
       "dependencies": {
         "wrappy": "1"
       }
@@ -4364,6 +4520,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/tunnel": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
+      "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
+      "engines": {
+        "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
+      }
+    },
     "node_modules/type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@@ -4400,12 +4564,28 @@
         "node": ">=14.17"
       }
     },
+    "node_modules/undici": {
+      "version": "5.28.3",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz",
+      "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==",
+      "dependencies": {
+        "@fastify/busboy": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.0"
+      }
+    },
     "node_modules/undici-types": {
       "version": "5.26.5",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
       "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
       "dev": true
     },
+    "node_modules/universal-user-agent": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",
+      "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ=="
+    },
     "node_modules/uri-js": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
@@ -4544,8 +4724,7 @@
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-      "dev": true
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
     "node_modules/ws": {
       "version": "7.5.9",
diff --git a/package.json b/package.json
index 3c7a04f..7bc548a 100644
--- a/package.json
+++ b/package.json
@@ -30,6 +30,7 @@
   "homepage": "https://github.com/autoblocksai/cli#readme",
   "dependencies": {
     "@actions/exec": "^1.1.1",
+    "@actions/github": "^6.0.0",
     "@autoblocks/client": "^0.0.33",
     "@hono/node-server": "^1.7.0",
     "@hono/zod-validator": "^0.1.11",
diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts
index 4196a63..44267ee 100644
--- a/src/handlers/testing/exec/index.ts
+++ b/src/handlers/testing/exec/index.ts
@@ -143,8 +143,7 @@ class RunManager {
       repositoryExternalId: ciContext.repoId,
       repositoryName: ciContext.repoName,
       repositoryHtmlUrl: ciContext.repoHtmlUrl,
-      // TODO: figure out how to get internal branch ID from github
-      branchExternalId: ciContext.branchName,
+      branchExternalId: ciContext.branchId,
       branchName: ciContext.branchName,
       isDefaultBranch: ciContext.branchName === ciContext.defaultBranchName,
       ciProvider: ciContext.ciProvider,
diff --git a/src/handlers/testing/exec/util/ci.ts b/src/handlers/testing/exec/util/ci.ts
index c2a2d81..30c8a41 100644
--- a/src/handlers/testing/exec/util/ci.ts
+++ b/src/handlers/testing/exec/util/ci.ts
@@ -1,5 +1,6 @@
 import { z } from 'zod';
 import { getExecOutput } from '@actions/exec';
+import github from '@actions/github';
 import fs from 'fs/promises';
 
 export interface CIContext {
@@ -7,6 +8,7 @@ export interface CIContext {
   repoId: string;
   repoName: string;
   repoHtmlUrl: string;
+  branchId: string;
   branchName: string;
   defaultBranchName: string;
   ciProvider: 'github';
@@ -24,6 +26,7 @@ export interface CIContext {
 
 const zGitHubEnvSchema = z.object({
   GITHUB_ACTIONS: z.literal('true'),
+  GITHUB_TOKEN: z.string(),
   GITHUB_SERVER_URL: z.string(),
   GITHUB_REPOSITORY: z.string(),
   GITHUB_REF_NAME: z.string(),
@@ -124,6 +127,7 @@ async function parseCommitFromGitLog(sha: string): Promise<Commit> {
 export async function makeCIContext(): Promise<CIContext> {
   const env = zGitHubEnvSchema.parse(process.env);
   const commit = await parseCommitFromGitLog(env.GITHUB_SHA);
+  const api = github.getOctokit(env.GITHUB_TOKEN);
 
   // GitHub Actions are triggered by webhook events, and the event payload is
   // stored in a JSON file at $GITHUB_EVENT_PATH.
@@ -134,16 +138,25 @@ export async function makeCIContext(): Promise<CIContext> {
   const repository = repositoryFromEvent(event.repository);
   const pullRequest = pullRequestFromEvent(event.pull_request);
 
+  // When it's a `push` event, the branch name is in `GITHUB_REF_NAME`, but on the `pull_request`
+  // event we want to use event.pull_request.head.ref, since `GITHUB_REF_NAME` will contain the
+  // name of the merge commit for the PR, like 5/merge.
+  const branchName = pullRequest?.head.ref || env.GITHUB_REF_NAME;
+
+  const { data: ref } = await api.rest.git.getRef({
+    owner: github.context.repo.owner,
+    repo: github.context.repo.repo,
+    ref: `heads/${branchName}`,
+  });
+
   return {
     gitProvider: 'github',
     repoId: repository.id,
     // full repo name including owner, e.g. 'octocat/Hello-World'
     repoName: env.GITHUB_REPOSITORY,
     repoHtmlUrl: [env.GITHUB_SERVER_URL, env.GITHUB_REPOSITORY].join('/'),
-    // When it's a `push` event, the branch name is in `GITHUB_REF_NAME`, but on the `pull_request`
-    // event we want to use event.pull_request.head.ref, since `GITHUB_REF_NAME` will contain the
-    // name of the merge commit for the PR, like 5/merge.
-    branchName: pullRequest?.head.ref || env.GITHUB_REF_NAME,
+    branchId: ref.node_id,
+    branchName,
     defaultBranchName: repository.default_branch,
     ciProvider: 'github',
     buildHtmlUrl: [

From 36c378cf5615848decc0cb7504941d088dfdb6cf Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Thu, 7 Mar 2024 18:11:54 -0500
Subject: [PATCH 3/9] Another commit

---
 src/handlers/testing/exec/util/ci.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/handlers/testing/exec/util/ci.ts b/src/handlers/testing/exec/util/ci.ts
index 30c8a41..03ee267 100644
--- a/src/handlers/testing/exec/util/ci.ts
+++ b/src/handlers/testing/exec/util/ci.ts
@@ -143,6 +143,7 @@ export async function makeCIContext(): Promise<CIContext> {
   // name of the merge commit for the PR, like 5/merge.
   const branchName = pullRequest?.head.ref || env.GITHUB_REF_NAME;
 
+  // Get the branch via the REST API so we can get its internal node ID
   const { data: ref } = await api.rest.git.getRef({
     owner: github.context.repo.owner,
     repo: github.context.repo.repo,

From a59a8f8d0de7933ab671d270b4da5454059c3dab Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Sun, 10 Mar 2024 17:22:20 -0400
Subject: [PATCH 4/9] ignore package-lock

---
 .prettierignore           | 1 +
 e2e/typescript/.gitignore | 4 ++++
 2 files changed, 5 insertions(+)
 create mode 100644 e2e/typescript/.gitignore

diff --git a/.prettierignore b/.prettierignore
index 03b4f5f..92bf10e 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -3,3 +3,4 @@
 CODEOWNERS
 *.sh
 *.txt
+.gitignore
diff --git a/e2e/typescript/.gitignore b/e2e/typescript/.gitignore
new file mode 100644
index 0000000..281c5c1
--- /dev/null
+++ b/e2e/typescript/.gitignore
@@ -0,0 +1,4 @@
+# We don't want a package-lock for this project because
+# we want to be able to install the latest version of
+# the dependencies when we run the tests.
+package-lock.json

From 4af706616da1d8155a56c84e835ad859c48cd14a Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Sun, 10 Mar 2024 17:24:46 -0400
Subject: [PATCH 5/9] Update error

---
 src/handlers/testing/exec/index.ts | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts
index 5681fe0..caf97e0 100644
--- a/src/handlers/testing/exec/index.ts
+++ b/src/handlers/testing/exec/index.ts
@@ -94,12 +94,11 @@ class RunManager {
         Authorization: `Bearer ${this.apiKey}`,
       },
     });
+    const data = await resp.json();
     if (!resp.ok) {
-      throw new Error(
-        `POST ${path} failed: ${resp.status} ${await resp.text()}`,
-      );
+      throw new Error(`POST ${subpath}${path} failed: ${JSON.stringify(data)}`);
     }
-    return resp.json();
+    return data;
   }
 
   private get isCI(): boolean {

From 360cdab90a88c492ecfbee5d4ec83625648052b0 Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Sun, 10 Mar 2024 17:27:26 -0400
Subject: [PATCH 6/9] typo

---
 src/handlers/testing/exec/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts
index caf97e0..7989539 100644
--- a/src/handlers/testing/exec/index.ts
+++ b/src/handlers/testing/exec/index.ts
@@ -149,7 +149,7 @@ class RunManager {
       buildHtmlUrl: ciContext.buildHtmlUrl,
       commitSha: ciContext.commitSha,
       commitMessage: ciContext.commitMessage,
-      commitComitterName: ciContext.commitCommitterName,
+      commitCommitterName: ciContext.commitCommitterName,
       commitCommitterEmail: ciContext.commitCommitterEmail,
       commitAuthorName: ciContext.commitAuthorName,
       commitAuthorEmail: ciContext.commitAuthorEmail,

From 0e68d2231474e632bede62752882f3f468d85140 Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Tue, 12 Mar 2024 13:43:57 -0400
Subject: [PATCH 7/9] add job names, drop link

---
 .github/workflows/e2e.yml                               | 4 ++++
 src/handlers/testing/exec/components/progress/index.tsx | 6 +-----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 17924d9..ea7b451 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -15,6 +15,8 @@ env:
 
 jobs:
   py:
+    name: python-e2e-${{ github.event_name }}
+
     runs-on: ubuntu-latest
 
     steps:
@@ -49,6 +51,8 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/e2e/python
 
   ts:
+    name: typescript-e2e-${{ github.event_name }}
+
     runs-on: ubuntu-latest
 
     steps:
diff --git a/src/handlers/testing/exec/components/progress/index.tsx b/src/handlers/testing/exec/components/progress/index.tsx
index b3b974a..d04c42c 100644
--- a/src/handlers/testing/exec/components/progress/index.tsx
+++ b/src/handlers/testing/exec/components/progress/index.tsx
@@ -1,4 +1,4 @@
-import { Box, Spacer, Static, Text, render } from 'ink';
+import { Box, Static, Text, render } from 'ink';
 import Spinner from 'ink-spinner';
 import { useEffect, useState } from 'react';
 import { EventName, emitter, type EventSchemas } from '../../emitter';
@@ -96,10 +96,6 @@ function TestRow(props: {
           <Spinner type="dots" />
         )}
         <Text bold={true}>{props.testExternalId}</Text>
-        <Spacer />
-        <Text>
-          {`https://app.autoblocks.ai/testing/local/tests/${encodeURIComponent(props.testExternalId)}`}
-        </Text>
       </Box>
       <Box paddingLeft={2} columnGap={2}>
         {props.runIsOver && testStatus === 'no-results' && (

From f67accf1808dc09c11a01014ed2287e81798a5b7 Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Tue, 12 Mar 2024 13:46:42 -0400
Subject: [PATCH 8/9] add back link

---
 .../testing/exec/components/progress/index.tsx        | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/handlers/testing/exec/components/progress/index.tsx b/src/handlers/testing/exec/components/progress/index.tsx
index d04c42c..70b00c1 100644
--- a/src/handlers/testing/exec/components/progress/index.tsx
+++ b/src/handlers/testing/exec/components/progress/index.tsx
@@ -1,4 +1,4 @@
-import { Box, Static, Text, render } from 'ink';
+import { Box, Spacer, Static, Text, render } from 'ink';
 import Spinner from 'ink-spinner';
 import { useEffect, useState } from 'react';
 import { EventName, emitter, type EventSchemas } from '../../emitter';
@@ -96,6 +96,15 @@ function TestRow(props: {
           <Spinner type="dots" />
         )}
         <Text bold={true}>{props.testExternalId}</Text>
+        {/* TODO: show URL for CI context as well */}
+        {!process.env.CI && (
+          <>
+            <Spacer />
+            <Text>
+              {`https://app.autoblocks.ai/testing/local/test/${encodeURIComponent(props.testExternalId)}`}
+            </Text>
+          </>
+        )}
       </Box>
       <Box paddingLeft={2} columnGap={2}>
         {props.runIsOver && testStatus === 'no-results' && (

From 7229e8ec390e52b71e7f4ec8066b7b1998e5b74f Mon Sep 17 00:00:00 2001
From: Nicole White <nicole@autoblocks.ai>
Date: Tue, 12 Mar 2024 23:06:44 -0400
Subject: [PATCH 9/9] add constants

---
 src/cli.ts                                              | 3 ++-
 src/handlers/testing/exec/components/progress/index.tsx | 3 ++-
 src/handlers/testing/exec/index.ts                      | 3 ++-
 src/util/constants.ts                                   | 2 ++
 4 files changed, 8 insertions(+), 3 deletions(-)
 create mode 100644 src/util/constants.ts

diff --git a/src/cli.ts b/src/cli.ts
index c44d163..f295788 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -5,6 +5,7 @@ import packageJson from '../package.json';
 import { renderOutdatedVersionComponent } from './components/outdated-version';
 import { handlers } from './handlers/index.js';
 import { AutoblocksTracer } from '@autoblocks/client';
+import { AUTOBLOCKS_WEBAPP_BASE_URL } from './util/constants';
 
 const packageName = packageJson.name;
 const packageVersion = packageJson.version;
@@ -46,7 +47,7 @@ const apiKeyMissingErrorMessage = `
 Autoblocks API key is required.
 Provide it via the AUTOBLOCKS_API_KEY environment variable or the --api-key option.
 
-You can get your API key from https://app.autoblocks.ai/settings/api-keys`;
+You can get your API key from ${AUTOBLOCKS_WEBAPP_BASE_URL}/settings/api-keys`;
 
 const parser = yargs(hideBin(process.argv))
   .command(
diff --git a/src/handlers/testing/exec/components/progress/index.tsx b/src/handlers/testing/exec/components/progress/index.tsx
index 70b00c1..fa5dd7c 100644
--- a/src/handlers/testing/exec/components/progress/index.tsx
+++ b/src/handlers/testing/exec/components/progress/index.tsx
@@ -2,6 +2,7 @@ import { Box, Spacer, Static, Text, render } from 'ink';
 import Spinner from 'ink-spinner';
 import { useEffect, useState } from 'react';
 import { EventName, emitter, type EventSchemas } from '../../emitter';
+import { AUTOBLOCKS_WEBAPP_BASE_URL } from '../../../../../util/constants';
 
 type ConsoleLog = EventSchemas[EventName.CONSOLE_LOG];
 type UncaughtError = EventSchemas[EventName.UNCAUGHT_ERROR];
@@ -101,7 +102,7 @@ function TestRow(props: {
           <>
             <Spacer />
             <Text>
-              {`https://app.autoblocks.ai/testing/local/test/${encodeURIComponent(props.testExternalId)}`}
+              {`${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/local/test/${encodeURIComponent(props.testExternalId)}`}
             </Text>
           </>
         )}
diff --git a/src/handlers/testing/exec/index.ts b/src/handlers/testing/exec/index.ts
index 7989539..d2912ea 100644
--- a/src/handlers/testing/exec/index.ts
+++ b/src/handlers/testing/exec/index.ts
@@ -9,6 +9,7 @@ import { renderTestProgress } from './components/progress';
 import { EventName, emitter, type EventSchemas } from './emitter';
 import { makeCIContext, type CIContext } from './util/ci';
 import { findAvailablePort } from './util/net';
+import { AUTOBLOCKS_API_BASE_URL } from '../../../util/constants';
 
 type UncaughtError = EventSchemas[EventName.UNCAUGHT_ERROR];
 
@@ -86,7 +87,7 @@ class RunManager {
 
   private async post<T>(path: string, body?: unknown): Promise<T> {
     const subpath = this.isCI ? '/testing/ci' : '/testing/local';
-    const resp = await fetch(`https://api.autoblocks.ai${subpath}${path}`, {
+    const resp = await fetch(`${AUTOBLOCKS_API_BASE_URL}${subpath}${path}`, {
       method: 'POST',
       body: body ? JSON.stringify(body) : undefined,
       headers: {
diff --git a/src/util/constants.ts b/src/util/constants.ts
new file mode 100644
index 0000000..cbd545e
--- /dev/null
+++ b/src/util/constants.ts
@@ -0,0 +1,2 @@
+export const AUTOBLOCKS_WEBAPP_BASE_URL = 'https://app.autoblocks.ai';
+export const AUTOBLOCKS_API_BASE_URL = 'https://api.autoblocks.ai';