Separate internal package for testcontainers so they can be used else…

…where

internal-packages/run-engine/package.json

@@ -5,6 +5,7 @@
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "dependencies": {
+    "@internal/testcontainers": "workspace:*",
     "@internal/zod-worker": "workspace:*",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/semantic-conventions": "^1.27.0",

internal-packages/run-engine/src/engine/index.test.ts

@@ -1,5 +1,5 @@
 import { expect } from "vitest";
-import { containerTest } from "../test/containerTest.js";
+import { containerTest } from "@internal/testcontainers";
 import { RunEngine } from "./index.js";
 import { PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database";
 

internal-packages/run-engine/src/run-queue/index.test.ts

@@ -1,7 +1,7 @@
 import { trace } from "@opentelemetry/api";
 import { Logger } from "@trigger.dev/core/logger";
 import { describe } from "node:test";
-import { redisTest } from "../test/containerTest.js";
+import { redisTest } from "@internal/testcontainers";
 import { RunQueue } from "./index.js";
 import { RunQueueShortKeyProducer } from "./keyProducer.js";
 import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js";

internal-packages/run-engine/tsconfig.json

@@ -15,6 +15,8 @@
     "noEmit": true,
     "strict": true,
     "paths": {
+      "@internal/testcontainers": ["../../internal-packages/testcontainers/src/index"],
+      "@internal/testcontainers/*": ["../../internal-packages/testcontainers/src/*"],
       "@internal/zod-worker": ["../../internal-packages/zod-worker/src/index"],
       "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"],
       "@trigger.dev/core": ["../../packages/core/src/index"],

internal-packages/testcontainers/README.md

@@ -0,0 +1,10 @@
+# Redis worker
+
+This is a simple worker that pulls tasks from a Redis queue (also in this package).
+
+Features
+
+- Configurable settings for concurrency and pull speed.
+- Job payloads.
+- A schema so only defined jobs can be added to the queue.
+- The ability to have future dates for jobs.

internal-packages/testcontainers/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@internal/testcontainers",
+  "private": true,
+  "version": "0.0.1",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@trigger.dev/database": "workspace:*",
+    "ioredis": "^5.3.2",
+    "typescript": "^4.8.4"
+  },
+  "devDependencies": {
+    "@testcontainers/postgresql": "^10.13.1",
+    "@testcontainers/redis": "^10.13.1",
+    "testcontainers": "^10.13.1",
+    "vitest": "^1.4.0"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit"
+  }
+}

internal-packages/testcontainers/src/index.ts

@@ -0,0 +1,67 @@
+import { StartedPostgreSqlContainer } from "@testcontainers/postgresql";
+import { StartedRedisContainer } from "@testcontainers/redis";
+import { Redis } from "ioredis";
+import { test } from "vitest";
+import { PrismaClient } from "@trigger.dev/database";
+import { createPostgresContainer, createRedisContainer } from "./utils";
+
+type PostgresContext = {
+  postgresContainer: StartedPostgreSqlContainer;
+  prisma: PrismaClient;
+};
+
+type RedisContext = { redisContainer: StartedRedisContainer; redis: Redis };
+type ContainerContext = PostgresContext & RedisContext;
+
+type Use<T> = (value: T) => Promise<void>;
+
+const postgresContainer = async ({}, use: Use<StartedPostgreSqlContainer>) => {
+  const { container } = await createPostgresContainer();
+  await use(container);
+  await container.stop();
+};
+
+const prisma = async (
+  { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer },
+  use: Use<PrismaClient>
+) => {
+  const prisma = new PrismaClient({
+    datasources: {
+      db: {
+        url: postgresContainer.getConnectionUri(),
+      },
+    },
+  });
+  await use(prisma);
+  await prisma.$disconnect();
+};
+
+export const postgresTest = test.extend<PostgresContext>({ postgresContainer, prisma });
+
+const redisContainer = async ({}, use: Use<StartedRedisContainer>) => {
+  const { container } = await createRedisContainer();
+  await use(container);
+  await container.stop();
+};
+
+const redis = async (
+  { redisContainer }: { redisContainer: StartedRedisContainer },
+  use: Use<Redis>
+) => {
+  const redis = new Redis({
+    host: redisContainer.getHost(),
+    port: redisContainer.getPort(),
+    password: redisContainer.getPassword(),
+  });
+  await use(redis);
+  await redis.quit();
+};
+
+export const redisTest = test.extend<RedisContext>({ redisContainer, redis });
+
+export const containerTest = test.extend<ContainerContext>({
+  postgresContainer,
+  prisma,
+  redisContainer,
+  redis,
+});

internal-packages/testcontainers/src/utils.ts

@@ -0,0 +1,35 @@
+import { PostgreSqlContainer } from "@testcontainers/postgresql";
+import { RedisContainer } from "@testcontainers/redis";
+import { execSync } from "child_process";
+import path from "path";
+
+export async function createPostgresContainer() {
+  const container = await new PostgreSqlContainer().start();
+
+  // Run migrations
+  const databasePath = path.resolve(__dirname, "../../../database");
+
+  execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, {
+    env: {
+      ...process.env,
+      DATABASE_URL: container.getConnectionUri(),
+      DIRECT_URL: container.getConnectionUri(),
+    },
+  });
+
+  // console.log(container.getConnectionUri());
+
+  return { url: container.getConnectionUri(), container };
+}
+
+export async function createRedisContainer() {
+  const container = await new RedisContainer().start();
+  try {
+    return {
+      container,
+    };
+  } catch (e) {
+    console.error(e);
+    throw e;
+  }
+}

internal-packages/testcontainers/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "ES2019",
+    "lib": ["ES2019", "DOM", "DOM.Iterable"],
+    "module": "CommonJS",
+    "moduleResolution": "Node10",
+    "moduleDetection": "force",
+    "verbatimModuleSyntax": false,
+    "types": ["vitest/globals"],
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "isolatedModules": true,
+    "preserveWatchOutput": true,
+    "skipLibCheck": true,
+    "noEmit": true,
+    "strict": true,
+    "paths": {
+      "@trigger.dev/core": ["../../packages/core/src/index"],
+      "@trigger.dev/core/*": ["../../packages/core/src/*"],
+      "@trigger.dev/database": ["../../internal-packages/database/src/index"],
+      "@trigger.dev/database/*": ["../../internal-packages/database/src/*"]
+    }
+  },
+  "exclude": ["node_modules"]
+}

internal-packages/testcontainers/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["**/*.test.ts"],
+    globals: true,
+  },
+});