feat: Run metadata (#1357)

* Run metadata * Remove metadata from context, move it to it’s own tab * More run metadata stuff - Add metadata to testing - Make using metadata outside of runs a no-op - Add docs * Replaying should copy over the metadata * transfer final attempt output to the task run * A couple of minor fixes * Use the new clientOrThrow() method everywhere * Cleaned up the update metadata endpoint and added an API doc page for it * Mirror task run attempt errors and output
triggerdotdev · Sep 26, 2024 · 3c492bc · 3c492bc
1 parent d74783c
commit 3c492bc
Show file tree

Hide file tree

Showing 48 changed files with 1,292 additions and 219 deletions.
diff --git a/.changeset/tasty-rats-rhyme.md b/.changeset/tasty-rats-rhyme.md
@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Add Run metadata to allow for storing up to 4KB of data on a run and update it during the run
diff --git a/apps/webapp/app/components/code/JSONEditor.tsx b/apps/webapp/app/components/code/JSONEditor.tsx
@@ -112,9 +112,9 @@ export function JSONEditor(opts: JSONEditorProps) {
   return (
     <div
       className={cn(
-        opts.className,
         "grid",
-        showButtons ? "grid-rows-[2.5rem_1fr]" : "grid-rows-[1fr]"
+        showButtons ? "grid-rows-[2.5rem_1fr]" : "grid-rows-[1fr]",
+        opts.className
       )}
     >
       {showButtons && (

diff --git a/apps/webapp/app/components/primitives/Tabs.tsx b/apps/webapp/app/components/primitives/Tabs.tsx
@@ -97,6 +97,7 @@ export function TabButton({
   return (
     <button
       className={cn("group flex flex-col items-center pt-1", props.className)}
+      type="button"
       ref={ref}
       {...props}
     >

diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts
@@ -212,6 +212,7 @@ const EnvironmentSchema = z.object({
   MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
   TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
   TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB
+  TASK_RUN_METADATA_MAXIMUM_SIZE: z.coerce.number().int().default(4_096), // 4KB
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

diff --git a/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts
@@ -29,6 +29,8 @@ const commonRunSelect = {
   completedAt: true,
   expiredAt: true,
   delayUntil: true,
+  metadata: true,
+  metadataType: true,
   ttl: true,
   tags: true,
   costInCents: true,
@@ -157,10 +159,8 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
         }
       }
 
-      const apiStatus = ApiRetrieveRunPresenter.apiStatusFromRunStatus(taskRun.status);
-
       return {
-        ...createCommonRunStructure(taskRun),
+        ...(await createCommonRunStructure(taskRun)),
         payload: $payload,
         payloadPresignedUrl: $payloadPresignedUrl,
         output: $output,
@@ -191,11 +191,15 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
               error: ApiRetrieveRunPresenter.apiErrorFromError(a.error),
             })),
         relatedRuns: {
-          root: taskRun.rootTaskRun ? createCommonRunStructure(taskRun.rootTaskRun) : undefined,
+          root: taskRun.rootTaskRun
+            ? await createCommonRunStructure(taskRun.rootTaskRun)
+            : undefined,
           parent: taskRun.parentTaskRun
-            ? createCommonRunStructure(taskRun.parentTaskRun)
+            ? await createCommonRunStructure(taskRun.parentTaskRun)
             : undefined,
-          children: taskRun.childRuns.map((r) => createCommonRunStructure(r)),
+          children: await Promise.all(
+            taskRun.childRuns.map(async (r) => await createCommonRunStructure(r))
+          ),
         },
       };
     });
@@ -329,7 +333,12 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
   }
 }
 
-function createCommonRunStructure(run: CommonRelatedRun) {
+async function createCommonRunStructure(run: CommonRelatedRun) {
+  const metadata = await parsePacket({
+    data: run.metadata ?? undefined,
+    dataType: run.metadataType,
+  });
+
   return {
     id: run.friendlyId,
     taskIdentifier: run.taskIdentifier,
@@ -354,6 +363,7 @@ function createCommonRunStructure(run: CommonRelatedRun) {
     ...ApiRetrieveRunPresenter.apiBooleanHelpersFromTaskRunStatus(run.status),
     triggerFunction: resolveTriggerFunction(run),
     batchId: run.batch?.friendlyId,
+    metadata,
   };
 }
 

diff --git a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts
@@ -1,4 +1,9 @@
-import { MachinePresetName, prettyPrintPacket, TaskRunError } from "@trigger.dev/core/v3";
+import {
+  MachinePresetName,
+  parsePacket,
+  prettyPrintPacket,
+  TaskRunError,
+} from "@trigger.dev/core/v3";
 import { RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { eventRepository } from "~/v3/eventRepository.server";
 import { machinePresetFromName } from "~/v3/machinePresets.server";
@@ -113,6 +118,8 @@ export class SpanPresenter extends BasePresenter {
         },
         payload: true,
         payloadType: true,
+        metadata: true,
+        metadataType: true,
         maxAttempts: true,
         project: {
           include: {
@@ -185,6 +192,10 @@ export class SpanPresenter extends BasePresenter {
 
     const span = await eventRepository.getSpan(spanId, run.traceId);
 
+    const metadata = run.metadata
+      ? await prettyPrintPacket(run.metadata, run.metadataType)
+      : undefined;
+
     const context = {
       task: {
         id: run.taskIdentifier,
@@ -272,6 +283,7 @@ export class SpanPresenter extends BasePresenter {
       error,
       links: span?.links,
       context: JSON.stringify(context, null, 2),
+      metadata,
     };
   }
 

diff --git a/apps/webapp/app/presenters/v3/TestTaskPresenter.server.ts b/apps/webapp/app/presenters/v3/TestTaskPresenter.server.ts
@@ -1,11 +1,6 @@
 import { ScheduledTaskPayload, parsePacket, prettyPrintPacket } from "@trigger.dev/core/v3";
-import {
-  RuntimeEnvironmentType,
-  TaskRunAttemptStatus,
-  TaskRunStatus,
-  TaskTriggerSource,
-} from "@trigger.dev/database";
-import { sqlDatabaseSchema, PrismaClient, prisma } from "~/db.server";
+import { RuntimeEnvironmentType, TaskRunStatus } from "@trigger.dev/database";
+import { PrismaClient, prisma, sqlDatabaseSchema } from "~/db.server";
 import { getTimezones } from "~/utils/timezones.server";
 import { getUsername } from "~/utils/username";
 
@@ -51,6 +46,8 @@ type RawRun = {
   payload: string;
   payloadType: string;
   runtimeEnvironmentId: string;
+  metadata?: string;
+  metadataType?: string;
 };
 
 export type StandardRun = Omit<RawRun, "number"> & {
@@ -131,6 +128,8 @@ export class TestTaskPresenter {
         taskr.status,
         taskr.payload,
         taskr."payloadType",
+        taskr.metadata,
+        taskr."metadataType",
         taskr."runtimeEnvironmentId"
     FROM 
         taskruns AS taskr
@@ -166,6 +165,9 @@ export class TestTaskPresenter {
                 ...r,
                 number,
                 payload: await prettyPrintPacket(r.payload, r.payloadType),
+                metadata: r.metadata
+                  ? await prettyPrintPacket(r.metadata, r.metadataType)
+                  : undefined,
               };
             })
           ),

diff --git a/...tes/_app.orgs.$organizationSlug.projects.v3.$projectParam.test.tasks.$taskParam/route.tsx b/...tes/_app.orgs.$organizationSlug.projects.v3.$projectParam.test.tasks.$taskParam/route.tsx
@@ -27,9 +27,11 @@ import {
   ResizablePanelGroup,
 } from "~/components/primitives/Resizable";
 import { Select } from "~/components/primitives/Select";
+import { TabButton, TabContainer } from "~/components/primitives/Tabs";
 import { TextLink } from "~/components/primitives/TextLink";
 import { TaskRunStatusCombo } from "~/components/runs/v3/TaskRunStatus";
 import { TimezoneList } from "~/components/scheduled/timezones";
+import { useSearchParams } from "~/hooks/useSearchParam";
 import { redirectBackWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
 import {
   ScheduledRun,
@@ -39,6 +41,7 @@ import {
 } from "~/presenters/v3/TestTaskPresenter.server";
 import { logger } from "~/services/logger.server";
 import { requireUserId } from "~/services/session.server";
+import { cn } from "~/utils/cn";
 import { docsPath, v3RunSpanPath, v3TaskParamsSchema } from "~/utils/pathBuilder";
 import { TestTaskService } from "~/v3/services/testTask.server";
 import { OutOfEntitlementError } from "~/v3/services/triggerTask.server";
@@ -129,27 +132,44 @@ export default function Page() {
 const startingJson = "{\n\n}";
 
 function StandardTaskForm({ task, runs }: { task: TestTask["task"]; runs: StandardRun[] }) {
+  const { value, replace } = useSearchParams();
+  const tab = value("tab");
+
   //form submission
   const submit = useSubmit();
   const lastSubmission = useActionData();
 
   //recent runs
   const [selectedCodeSampleId, setSelectedCodeSampleId] = useState(runs.at(0)?.id);
-  const selectedCodeSample = runs.find((r) => r.id === selectedCodeSampleId)?.payload;
+  const selectedCodeSample = runs.find((r) => r.id === selectedCodeSampleId);
+  const selectedCodeSamplePayload = selectedCodeSample?.payload;
+  const selectedCodeSampleMetadata = selectedCodeSample?.metadata;
 
-  const [defaultJson, setDefaultJson] = useState<string>(selectedCodeSample ?? startingJson);
-  const setCode = useCallback((code: string) => {
-    setDefaultJson(code);
+  const [defaultPayloadJson, setDefaultPayloadJson] = useState<string>(
+    selectedCodeSamplePayload ?? startingJson
+  );
+  const setPayload = useCallback((code: string) => {
+    setDefaultPayloadJson(code);
   }, []);
 
-  const currentJson = useRef<string>(defaultJson);
+  const currentPayloadJson = useRef<string>(defaultPayloadJson);
+
+  const [defaultMetadataJson, setDefaultMetadataJson] = useState<string>(
+    selectedCodeSampleMetadata ?? "{}"
+  );
+  const setMetadata = useCallback((code: string) => {
+    setDefaultMetadataJson(code);
+  }, []);
+
+  const currentMetadataJson = useRef<string>(defaultMetadataJson);
 
   const submitForm = useCallback(
     (e: React.FormEvent<HTMLFormElement>) => {
       submit(
         {
           triggerSource: "STANDARD",
-          payload: currentJson.current,
+          payload: currentPayloadJson.current,
+          metadata: currentMetadataJson.current,
           taskIdentifier: task.taskIdentifier,
           environmentId: task.environment.id,
         },
@@ -160,7 +180,7 @@ function StandardTaskForm({ task, runs }: { task: TestTask["task"]; runs: Standa
       );
       e.preventDefault();
     },
-    [currentJson]
+    [currentPayloadJson, currentMetadataJson]
   );
 
   const [form, { environmentId, payload }] = useForm({
@@ -183,28 +203,73 @@ function StandardTaskForm({ task, runs }: { task: TestTask["task"]; runs: Standa
       <ResizablePanelGroup orientation="horizontal">
         <ResizablePanel id="test-task-main" min="100px" default="60%">
           <div className="h-full bg-charcoal-900">
-            <JSONEditor
-              defaultValue={defaultJson}
-              readOnly={false}
-              basicSetup
-              onChange={(v) => {
-                currentJson.current = v;
-
-                //deselect the example if it's been edited
-                if (selectedCodeSampleId) {
-                  if (v !== selectedCodeSample) {
-                    setDefaultJson(v);
-                    setSelectedCodeSampleId(undefined);
+            <TabContainer className="px-3 pt-2">
+              <TabButton
+                isActive={!tab || tab === "payload"}
+                layoutId="test-editor"
+                onClick={() => {
+                  replace({ tab: "payload" });
+                }}
+              >
+                Payload
+              </TabButton>
+
+              <TabButton
+                isActive={tab === "metadata"}
+                layoutId="test-editor"
+                onClick={() => {
+                  replace({ tab: "metadata" });
+                }}
+              >
+                Metadata
+              </TabButton>
+            </TabContainer>
+            <div>
+              <JSONEditor
+                defaultValue={defaultPayloadJson}
+                readOnly={false}
+                basicSetup
+                onChange={(v) => {
+                  currentPayloadJson.current = v;
+
+                  //deselect the example if it's been edited
+                  if (selectedCodeSampleId) {
+                    if (v !== selectedCodeSamplePayload) {
+                      setDefaultPayloadJson(v);
+                      setSelectedCodeSampleId(undefined);
+                    }
                   }
-                }
-              }}
-              height="100%"
-              min-height="100%"
-              max-height="100%"
-              autoFocus
-              placeholder="Use your schema to enter valid JSON or add one of the recent payloads then click 'Run test'"
-              className="h-full"
-            />
+                }}
+                height="100%"
+                min-height="100%"
+                max-height="100%"
+                autoFocus={!tab || tab === "payload"}
+                placeholder="{ }"
+                className={cn("h-full", tab === "metadata" && "hidden")}
+              />
+              <JSONEditor
+                defaultValue={defaultMetadataJson}
+                readOnly={false}
+                basicSetup
+                onChange={(v) => {
+                  currentMetadataJson.current = v;
+
+                  //deselect the example if it's been edited
+                  if (selectedCodeSampleId) {
+                    if (v !== selectedCodeSampleMetadata) {
+                      setDefaultMetadataJson(v);
+                      setSelectedCodeSampleId(undefined);
+                    }
+                  }
+                }}
+                height="100%"
+                min-height="100%"
+                max-height="100%"
+                autoFocus={tab === "metadata"}
+                placeholder=""
+                className={cn("h-full", tab !== "metadata" && "hidden")}
+              />
+            </div>
           </div>
         </ResizablePanel>
         <ResizableHandle id="test-task-handle" />
@@ -213,9 +278,10 @@ function StandardTaskForm({ task, runs }: { task: TestTask["task"]; runs: Standa
             runs={runs}
             selectedId={selectedCodeSampleId}
             onSelected={(id) => {
-              const payload = runs.find((r) => r.id === id)?.payload;
-              if (!payload) return;
-              setCode(payload);
+              const run = runs.find((r) => r.id === id);
+              if (!run) return;
+              setPayload(run.payload);
+              run.metadata && setMetadata(run.metadata);
               setSelectedCodeSampleId(id);
             }}
           />

diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.test/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.test/route.tsx
@@ -75,8 +75,13 @@ export default function Page() {
   const navigation = useNavigation();
 
   const location = useLocation();
+  const locationSearchParams = new URLSearchParams(location.search);
+  const navigationSearchParams = new URLSearchParams(navigation.location?.search);
+
   const isLoadingTasks =
-    navigation.state === "loading" && navigation.location.pathname === location.pathname;
+    navigation.state === "loading" &&
+    navigation.location.pathname === location.pathname &&
+    navigationSearchParams.get("environment") !== locationSearchParams.get("environment");
 
   return (
     <PageContainer>