feat: dashboard improvement with animated screenshots

OpenAdaptAI · Nov 16, 2024 · 41f252b · 41f252b
1 parent c916541
commit 41f252b
Show file tree

Hide file tree

Showing 9 changed files with 766 additions and 52 deletions.
diff --git a/openadapt/app/dashboard/api/recordings.py b/openadapt/app/dashboard/api/recordings.py
@@ -29,6 +29,9 @@ def attach_routes(self) -> APIRouter:
         self.app.add_api_route("/start", self.start_recording)
         self.app.add_api_route("/stop", self.stop_recording)
         self.app.add_api_route("/status", self.recording_status)
+        self.app.add_api_route(
+            "/{recording_id}/screenshots", self.get_recording_screenshots
+        )
         self.recording_detail_route()
         return self.app
 
@@ -63,6 +66,25 @@ def recording_status() -> dict[str, bool]:
         """Get the recording status."""
         return {"recording": cards.is_recording()}
 
+    @staticmethod
+    def get_recording_screenshots(recording_id: int) -> dict[str, list[str]]:
+        """Get all screenshots for a specific recording."""
+        session = crud.get_new_session(read_only=True)
+        recording = crud.get_recording_by_id(session, recording_id)
+        action_events = get_events(session, recording)
+
+        screenshots = []
+        for action_event in action_events:
+            try:
+                image = display_event(action_event)
+                if image:
+                    screenshots.append(image2utf8(image))
+            except Exception as e:
+                logger.info(f"Failed to display event: {e}")
+                continue
+
+        return {"screenshots": screenshots}
+
     def recording_detail_route(self) -> None:
         """Add the recording detail route as a websocket."""
 

diff --git a/openadapt/app/dashboard/app/recordings/RawRecordings.tsx b/openadapt/app/dashboard/app/recordings/RawRecordings.tsx
@@ -1,21 +1,59 @@
 import { SimpleTable } from '@/components/SimpleTable';
+import dynamic from 'next/dynamic';
 import { Recording } from '@/types/recording';
-import React, { useEffect, useState } from 'react'
+import React, { useEffect, useState } from 'react';
 import { timeStampToDateString } from '../utils';
 import { useRouter } from 'next/navigation';
 
+const FramePlayer = dynamic<{ recording: Recording; frameRate: number }>(
+    () => import('@/components/FramePlayer').then((mod) => mod.FramePlayer),
+    { ssr: false }
+);
+
+async function fetchRecordingWithScreenshots(recordingId: string | number) {
+    try {
+        const response = await fetch(`/api/recordings/${recordingId}/screenshots`);
+        if (!response.ok) {
+            throw new Error('Failed to fetch screenshots');
+        }
+        const data = await response.json();
+        return data.screenshots || [];
+    } catch (error) {
+        console.error('Error fetching screenshots:', error);
+        return [];
+    }
+}
+
 export const RawRecordings = () => {
     const [recordings, setRecordings] = useState<Recording[]>([]);
+    const [loading, setLoading] = useState(true);
     const router = useRouter();
 
-    function fetchRecordings() {
-        fetch('/api/recordings').then(res => {
+    async function fetchRecordings() {
+        try {
+            setLoading(true);
+            const res = await fetch('/api/recordings');
             if (res.ok) {
-                res.json().then((data) => {
-                    setRecordings(data.recordings);
-                });
+                const data = await res.json();
+
+                // Fetch screenshots for all recordings in parallel
+                const recordingsWithScreenshots = await Promise.all(
+                    data.recordings.map(async (rec: Recording) => {
+                        const screenshots = await fetchRecordingWithScreenshots(rec.id);
+                        return {
+                            ...rec,
+                            screenshots
+                        };
+                    })
+                );
+
+                setRecordings(recordingsWithScreenshots);
             }
-        })
+        } catch (error) {
+            console.error('Error fetching recordings:', error);
+        } finally {
+            setLoading(false);
+        }
     }
 
     useEffect(() => {
@@ -26,19 +64,38 @@ export const RawRecordings = () => {
         return () => router.push(`/recordings/detail/?id=${recording.id}`);
     }
 
+    if (loading) {
+        return <div className="text-center py-4">Loading recordings...</div>;
+    }
+
     return (
         <SimpleTable
             columns={[
                 {name: 'ID', accessor: 'id'},
                 {name: 'Description', accessor: 'task_description'},
-                {name: 'Start time', accessor: (recording: Recording) => recording.video_start_time ? timeStampToDateString(recording.video_start_time) : 'N/A'},
-                // {name: 'Timestamp', accessor: (recording: Recording) => recording.timestamp ? timeStampToDateString(recording.timestamp) : 'N/A'},
-                {name: 'Monitor Width/Height', accessor: (recording: Recording) => `${recording.monitor_width}/${recording.monitor_height}`},
-                // {name: 'Double Click Interval Seconds/Pixels', accessor: (recording: Recording) => `${recording.double_click_interval_seconds}/${recording.double_click_distance_pixels}`},
+                {name: 'Start time', accessor: (recording: Recording) =>
+                    recording.video_start_time
+                        ? timeStampToDateString(recording.video_start_time)
+                        : 'N/A'
+                },
+                {name: 'Monitor Width/Height', accessor: (recording: Recording) =>
+                    `${recording.monitor_width}/${recording.monitor_height}`
+                },
+                {
+                    name: 'Video',
+                    accessor: (recording: Recording) => (
+                        <div className="min-w-[200px]">
+                            <FramePlayer
+                                recording={recording}
+                                frameRate={1}
+                            />
+                        </div>
+                    ),
+                }
             ]}
             data={recordings}
             refreshData={fetchRecordings}
             onClickRow={onClickRow}
         />
-    )
-}
+    );
+};
diff --git a/openadapt/app/dashboard/app/settings/(api_keys)/form.tsx b/openadapt/app/dashboard/app/settings/(api_keys)/form.tsx
@@ -25,7 +25,7 @@ export const Form = ({ settings }: Props) => {
     const inputClasses = "w-full px-3 py-2 border border-gray-200 rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500 outline-none transition-all duration-200 bg-white";
     const labelClasses = "block text-sm font-medium text-gray-700 mb-1";
     const fieldsetClasses = "border border-gray-200 rounded-xl p-6 bg-zinc-100 shadow-lg relative mt-2";
-    const legendClasses = "absolute -top-3 bg-white px-2 text-sm font-semibold text-gray-800 shadow-lg rounded-lg";
+    const legendClasses = "absolute -top-3 bg-primary/80 px-2 text-sm font-semibold text-zinc-200 shadow-lg rounded-lg";
 
     return (
         <form onSubmit={form.onSubmit(saveSettings(form))} className="max-w-6xl mx-auto p-6">

diff --git a/openadapt/app/dashboard/components/FramePlayer/FramePlayer.tsx b/openadapt/app/dashboard/components/FramePlayer/FramePlayer.tsx
@@ -0,0 +1,120 @@
+"use client"
+
+import { Recording } from '@/types/recording';
+import React, { useEffect, useRef, useState } from "react";
+import { Stage, Layer, Image as KonvaImage, Group, Rect } from "react-konva";
+import Konva from "konva";
+
+interface FramePlayerProps {
+  recording: Recording;
+  frameRate: number;
+}
+
+export const FramePlayer: React.FC<FramePlayerProps> = ({ recording, frameRate }) => {
+  const [currentScreenshotIndex, setCurrentScreenshotIndex] = useState(0);
+  const [currentImage, setCurrentImage] = useState<HTMLImageElement | null>(null);
+  const [isHovering, setIsHovering] = useState(false);
+  const imageRef = useRef<Konva.Image>(null);
+  const intervalRef = useRef<NodeJS.Timeout | null>(null);
+
+  const STAGE_WIDTH = 300;
+  const STAGE_HEIGHT = 200;
+  const CORNER_RADIUS = 8;
+
+  // Load initial image
+  useEffect(() => {
+    if (!recording?.screenshots?.length) return;
+
+    const img = new window.Image();
+    img.src = recording.screenshots[0];
+    img.onload = () => {
+      setCurrentImage(img);
+    };
+  }, [recording]);
+
+  useEffect(() => {
+    if (!recording?.screenshots?.length ) {
+      if (isHovering && currentScreenshotIndex !== 0) {
+        setCurrentScreenshotIndex(0);
+      }
+      return;
+    }
+
+    intervalRef.current = setInterval(() => {
+      setCurrentScreenshotIndex((prevIndex) => {
+        return (prevIndex + 1) % recording.screenshots.length;
+      });
+    }, 1000 / frameRate);
+
+    return () => {
+      if (intervalRef.current) {
+        clearInterval(intervalRef.current);
+      }
+    };
+  }, [recording, frameRate, isHovering]);
+
+  // Handle image loading when screenshot index changes
+  useEffect(() => {
+    if (!recording?.screenshots?.length) return;
+
+    const img = new window.Image();
+    img.src = recording.screenshots[currentScreenshotIndex];
+    img.onload = () => {
+      setCurrentImage(img);
+    };
+  }, [currentScreenshotIndex, recording]);
+
+  if (!recording?.screenshots?.length) {
+    return (
+      <div className="flex items-center justify-center w-full h-[150px] bg-gray-100 rounded-lg">
+        <span className="text-sm text-gray-500">No screenshots</span>
+      </div>
+    );
+  }
+
+  return (
+    <div
+      className="relative cursor-pointer"
+      onMouseEnter={() => setIsHovering(true)}
+      onMouseLeave={() => {
+        setIsHovering(false);
+        setCurrentScreenshotIndex(0);
+      }}
+    >
+      <Stage width={STAGE_WIDTH} height={STAGE_HEIGHT}>
+        <Layer>
+          <Group
+            clipFunc={(ctx) => {
+              ctx.beginPath();
+              ctx.moveTo(CORNER_RADIUS, 0);
+              ctx.lineTo(STAGE_WIDTH - CORNER_RADIUS, 0);
+              ctx.quadraticCurveTo(STAGE_WIDTH, 0, STAGE_WIDTH, CORNER_RADIUS);
+              ctx.lineTo(STAGE_WIDTH, STAGE_HEIGHT - CORNER_RADIUS);
+              ctx.quadraticCurveTo(STAGE_WIDTH, STAGE_HEIGHT, STAGE_WIDTH - CORNER_RADIUS, STAGE_HEIGHT);
+              ctx.lineTo(CORNER_RADIUS, STAGE_HEIGHT);
+              ctx.quadraticCurveTo(0, STAGE_HEIGHT, 0, STAGE_HEIGHT - CORNER_RADIUS);
+              ctx.lineTo(0, CORNER_RADIUS);
+              ctx.quadraticCurveTo(0, 0, CORNER_RADIUS, 0);
+              ctx.closePath();
+            }}
+          >
+            {currentImage && (
+              <KonvaImage
+                ref={imageRef}
+                image={currentImage}
+                width={STAGE_WIDTH}
+                height={STAGE_HEIGHT}
+                listening={false}
+              />
+            )}
+          </Group>
+        </Layer>
+      </Stage>
+      {!isHovering && (
+        <div className="absolute bottom-1 left-1 right-1 flex justify-between text-xs text-white bg-black/50 px-2 py-0.5 rounded w-fit">
+          <span>Frame {currentScreenshotIndex + 1}/{recording.screenshots.length}</span>
+        </div>
+      )}
+    </div>
+  );
+};
diff --git a/openadapt/app/dashboard/components/FramePlayer/index.tsx b/openadapt/app/dashboard/components/FramePlayer/index.tsx
@@ -0,0 +1 @@
+export { FramePlayer } from './FramePlayer'
diff --git a/openadapt/app/dashboard/next.config.js b/openadapt/app/dashboard/next.config.js
@@ -1,36 +1,48 @@
 /** @type {import('next').NextConfig} */
-// get values from environment variables
-const { DASHBOARD_SERVER_PORT } = process.env
+// Get values from environment variables
+const { DASHBOARD_SERVER_PORT } = process.env;
 
 const nextConfig = {
-    rewrites: async () => {
-        return [
-            {
-                source: '/',
-                destination: `/recordings`,
-            },
-            {
-                source: '/api/:path*',
-                destination: `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/api/:path*`
-            },
-            {
-                source: '/docs',
-                destination:
-                    process.env.NODE_ENV === 'development'
-                        ? `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/docs`
-                        : '/api/docs',
-            },
-            {
-                source: '/openapi.json',
-                destination:
-                    process.env.NODE_ENV === 'development'
-                        ? `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/openapi.json`
-                        : '/api/openapi.json',
-            },
-        ]
-    },
-    output: 'export',
-    reactStrictMode: false,
-}
+  async rewrites() {
+    return [
+      {
+        source: '/',
+        destination: `/recordings`,
+      },
+      {
+        source: '/api/:path*',
+        destination: `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/api/:path*`,
+      },
+      {
+        source: '/docs',
+        destination:
+          process.env.NODE_ENV === 'development'
+            ? `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/docs`
+            : '/api/docs',
+      },
+      {
+        source: '/openapi.json',
+        destination:
+          process.env.NODE_ENV === 'development'
+            ? `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/openapi.json`
+            : '/api/openapi.json',
+      },
+    ];
+  },
 
-module.exports = nextConfig
+  webpack: (config, { isServer }) => {
+    if (!isServer) {
+      // Prevent `canvas` from being bundled on the client side
+      config.resolve.alias['canvas'] = false;
+    } else {
+      // Exclude canvas from the server bundle, treating it as an external dependency for Node
+      config.externals.push({ canvas: 'commonjs canvas' });
+    }
+    return config;
+  },
+
+  output: 'export',
+  reactStrictMode: false,
+};
+
+module.exports = nextConfig;