EPD-621 Updates for CI

.github/workflows/ci.yml

@@ -8,6 +8,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+  TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
+
 jobs:
   ci:
     runs-on: ubuntu-latest
@@ -46,10 +50,6 @@ jobs:
 
       - name: Run build
         run: npm run build
-        env:
-          TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
 
       - name: Run testing exec
         run: npx autoblocks testing exec -- echo "hi"
-        env:
-          AUTOBLOCKS_API_KEY: test

.github/workflows/e2e.yml

@@ -0,0 +1,68 @@
+name: E2E
+
+on:
+  # Run on different types of events to ensure we are
+  # handling the git data correctly in each scenario
+  push:
+  pull_request:
+  schedule:
+    - cron: '17 15 * * *'
+
+env:
+  AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
+  TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
+
+jobs:
+  py:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install CLI dependencies
+        run: npm ci
+
+      - name: Build CLI
+        run: npm run build
+
+      - name: Install dependencies in e2e/python
+        run: pip install -r requirements.txt
+        working-directory: e2e/python
+
+      - name: Run tests in e2e/python
+        run: ../../bin/cli.js testing exec -- python3 run.py
+        working-directory: e2e/python
+        env:
+          PYTHONPATH: ${{ github.workspace }}/e2e/python
+
+  ts:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install CLI dependencies
+        run: npm ci
+
+      - name: Build CLI
+        run: npm run build
+
+      - name: Install dependencies in e2e/typescript
+        run: npm install
+        working-directory: e2e/typescript
+
+      - name: Run tests in e2e/typescript
+        run: ../../bin/cli.js testing exec -- npx tsx run.ts
+        working-directory: e2e/typescript

.prettierignore

@@ -2,3 +2,4 @@
 *.py
 CODEOWNERS
 *.sh
+*.txt

e2e/python/requirements.txt

@@ -0,0 +1 @@
+autoblocksai

e2e/python/run.py

@@ -0,0 +1,80 @@
+import uuid
+import random
+import asyncio
+import dataclasses
+
+from autoblocks.testing.models import BaseTestCase
+from autoblocks.testing.models import BaseTestEvaluator
+from autoblocks.testing.models import Evaluation
+from autoblocks.testing.models import Threshold
+from autoblocks.testing.util import md5
+from autoblocks.testing.run import run_test_suite
+
+
+@dataclasses.dataclass
+class MyTestCase(BaseTestCase):
+    input: str
+    expected_substrings: list[str]
+
+    def hash(self) -> str:
+        return md5(self.input)
+
+
+async def test_fn(test_case: MyTestCase) -> str:
+    await asyncio.sleep(random.random())
+
+    substrings = test_case.input.split("-")
+    if random.random() < 0.2:
+        substrings.pop()
+
+    return "-".join(substrings)
+
+
+class HasAllSubstrings(BaseTestEvaluator):
+    id = "has-all-substrings"
+
+    def evaluate_test_case(self, test_case: MyTestCase, output: str) -> Evaluation:
+        score = 1 if all(s in output for s in test_case.expected_substrings) else 0
+        return Evaluation(
+            score=score,
+            threshold=Threshold(gte=1),
+        )
+
+
+class IsFriendly(BaseTestEvaluator):
+    id = "is-friendly"
+
+    async def get_score(self, output: str) -> float:
+        await asyncio.sleep(random.random())
+        return random.random()
+
+    async def evaluate_test_case(self, test_case: BaseTestCase, output: str) -> Evaluation:
+        score = await self.get_score(output)
+        return Evaluation(
+            score=score,
+        )
+
+
+def gen_test_cases(n: int) -> list[MyTestCase]:
+    test_cases = []
+    for _ in range(n):
+        random_id = str(uuid.uuid4())
+        test_cases.append(
+            MyTestCase(
+                input=random_id,
+                expected_substrings=random_id.split("-"),
+            ),
+        )
+    return test_cases
+
+
+if __name__ == "__main__":
+  run_test_suite(
+      id="my-test-suite",
+      fn=test_fn,
+      test_cases=gen_test_cases(40),
+      evaluators=[
+          HasAllSubstrings(),
+          IsFriendly(),
+      ],
+  )

e2e/typescript/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "ts-e2e",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "@autoblocks/client": "*",
+    "typescript": "*"
+  }
+}

e2e/typescript/run.ts

@@ -0,0 +1,88 @@
+import {
+  BaseTestEvaluator,
+  runTestSuite,
+  type Evaluation,
+} from '@autoblocks/client/testing';
+import * as crypto from 'crypto';
+
+interface MyTestCase {
+  input: string;
+  expectedSubstrings: string[];
+}
+
+async function testFn({ testCase }: { testCase: MyTestCase }): Promise<string> {
+  // Simulate doing work
+  await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
+
+  const substrings = testCase.input.split('-');
+  if (Math.random() < 0.2) {
+    // Remove a substring randomly. This will cause about 20% of the test cases to fail
+    // the "has-all-substrings" evaluator.
+    substrings.pop();
+  }
+
+  return substrings.join('-');
+}
+
+class HasAllSubstrings extends BaseTestEvaluator<MyTestCase, string> {
+  id = 'has-all-substrings';
+
+  evaluateTestCase(args: { testCase: MyTestCase; output: string }): Evaluation {
+    const score = args.testCase.expectedSubstrings.every((s) =>
+      args.output.includes(s),
+    )
+      ? 1
+      : 0;
+
+    return {
+      score,
+      threshold: {
+        gte: 1,
+      },
+    };
+  }
+}
+
+class IsFriendly extends BaseTestEvaluator<MyTestCase, string> {
+  id = 'is-friendly';
+
+  async getScore(output: string): Promise<number> {
+    // eslint-disable-next-line no-console
+    console.log(`Determining score for output: ${output}`);
+    await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
+    return Math.random();
+  }
+
+  async evaluateTestCase(args: {
+    testCase: MyTestCase;
+    output: string;
+  }): Promise<Evaluation> {
+    const score = await this.getScore(args.output);
+
+    return {
+      score,
+    };
+  }
+}
+
+function genTestCases(n: number): MyTestCase[] {
+  const testCases: MyTestCase[] = [];
+  for (let i = 0; i < n; i++) {
+    const randomId = crypto.randomUUID();
+    testCases.push({
+      input: randomId,
+      expectedSubstrings: randomId.split('-'),
+    });
+  }
+  return testCases;
+}
+
+(async () => {
+  await runTestSuite<MyTestCase, string>({
+    id: 'my-test-suite',
+    fn: testFn,
+    testCaseHash: ['input'],
+    testCases: genTestCases(40),
+    evaluators: [new HasAllSubstrings(), new IsFriendly()],
+  });
+})();

e2e/typescript/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "moduleResolution": "node",
+    "module": "esnext",
+    "target": "esnext"
+  },
+  "include": ["**/*.ts"]
+}

package.json

@@ -30,7 +30,7 @@
   "homepage": "https://github.com/autoblocksai/cli#readme",
   "dependencies": {
     "@actions/exec": "^1.1.1",
-    "@autoblocks/client": "^0.0.32",
+    "@autoblocks/client": "^0.0.33",
     "@hono/node-server": "^1.7.0",
     "@hono/zod-validator": "^0.1.11",
     "hono": "^3.12.12",

src/handlers/testing/exec/emitter.ts

@@ -12,7 +12,7 @@ export enum EventName {
 
 // Zod schemas for event data
 const zConsoleLogSchema = z.object({
-  ctx: z.enum(['cmd', 'cli', 'cli-server']),
+  ctx: z.enum(['cmd', 'cli']),
   level: z.enum(['debug', 'info', 'warn', 'error']),
   message: z.string(),
 });