added amicaHandler

src/features/chat/chat.ts

@@ -20,13 +20,14 @@ import { localXTTSTTS} from "@/features/localXTTS/localXTTS";
 
 import { AmicaLife } from '@/features/amicaLife/amicaLife';
 
-import { config } from "@/utils/config";
+import { config, updateConfig } from "@/utils/config";
 import { cleanTalk } from "@/utils/cleanTalk";
 import { processResponse } from "@/utils/processResponse";
 import { wait } from "@/utils/wait";
 import { isCharacterIdle, characterIdleTime, resetIdleTimer } from "@/utils/isIdle";
 import { getOpenRouterChatResponseStream } from './openRouterChat';
 import { handleUserInput } from '../externalAPI/externalAPI';
+import { loadVRMAnimation } from '@/lib/VRMAnimation/loadVRMAnimation';
 
 
 type Speak = {
@@ -124,6 +125,8 @@ export class Chat {
 
     this.updateAwake();
     this.initialized = true;
+
+    this.serverSentEvent();
   }
 
   public setMessageList(messages: Message[]) {
@@ -359,6 +362,100 @@ export class Chat {
     await this.makeAndHandleStream(messages);
   }
 
+  public serverSentEvent() {
+    // Client-side code in a React component or elsewhere
+    const eventSource = new EventSource('/api/amicaHandler');
+
+    eventSource.onmessage = async (event) => {
+      const data = JSON.parse(event.data);
+      console.log('Received message:', data);
+
+    };
+    // Listen for incoming messages from the server
+    eventSource.onmessage = async (event) => {
+      try {
+        // Parse the incoming JSON message
+        const message = JSON.parse(event.data);
+
+        console.log(message);
+
+        // Destructure to get the message type and data
+        const { type, data } = message;
+
+        // Handle the message based on its type
+        switch (type) {
+          case 'normal':
+            console.log('Normal message received:', data);
+            const messages: Message[] = [
+              { role: "system", content: config("system_prompt") },
+              ...this.messageList!,
+              { role: "user", content: data},
+            ];
+            let stream = await getEchoChatResponseStream(messages);
+            this.streams.push(stream);
+            this.handleChatResponseStream();
+            break;
+
+          case 'animation':
+            console.log('Animation data received:', data);
+            const animation = await loadVRMAnimation(`/animations/${data}`);
+            if (!animation) {
+              throw new Error("Loading animation failed");
+            }
+            this.viewer?.model?.playAnimation(animation,data);
+            requestAnimationFrame(() => { this.viewer?.resetCameraLerp(); });
+            break;
+
+          case 'playback':
+            console.log('Playback flag received:', data);
+            this.viewer?.startRecording();
+            // Automatically stop recording after 10 seconds
+            setTimeout(() => {
+              this.viewer?.stopRecording((videoBlob) => {
+                // Log video blob to console
+                console.log("Video recording finished", videoBlob);
+
+                // Create a download link for the video file
+                const url = URL.createObjectURL(videoBlob!);
+                const a = document.createElement("a");
+                a.href = url;
+                a.download = "recording.webm"; // Set the file name for download
+                document.body.appendChild(a);
+                a.click();
+                document.body.removeChild(a);
+
+                // Revoke the URL to free up memory
+                URL.revokeObjectURL(url);
+              });
+            }, data); // Stop recording after 10 seconds
+            break;
+
+          case 'systemPrompt':
+            console.log('System Prompt data received:', data);
+            updateConfig("system_prompt",data);
+            break;
+
+          default:
+            console.warn('Unknown message type:', type);
+        }
+      } catch (error) {
+        console.error('Error parsing SSE message:', error);
+      }
+    };
+
+
+    eventSource.addEventListener('end', () => {
+      console.log('SSE session ended');
+      eventSource.close();
+    });
+
+    eventSource.onerror = (error) => {
+      console.error('Error in SSE connection:', error);
+      eventSource.close();
+    };
+
+  }
+
 
   public async makeAndHandleStream(messages: Message[]) {
     try {

src/features/vrmViewer/viewer.ts

@@ -26,6 +26,10 @@ export class Viewer {
   private sendScreenshotToCallback: boolean;
   private screenshotCallback: BlobCallback | undefined;
 
+  private mediaRecorder?: MediaRecorder;
+  private recordedChunks: Blob[] = [];
+  private videoStream: any;
+
   constructor() {
     this.isReady = false;
     this.sendScreenshotToCallback = false;
@@ -218,11 +222,78 @@ export class Viewer {
       if (this.sendScreenshotToCallback && this.screenshotCallback) {
         this._renderer.domElement.toBlob(this.screenshotCallback, "image/jpeg");
         this.sendScreenshotToCallback = false;
-
       }
     }
   };
 
+  public startStreaming(videoElement: HTMLVideoElement) {
+    if (!this._renderer) return;
+
+    // Create a stream from the renderer's canvas
+    const stream = this._renderer.domElement.captureStream(60); // 60 FPS for smooth streaming
+
+    this.videoStream = stream;
+
+    // Assign the stream to the provided video element for live view
+    videoElement.srcObject = stream;
+    videoElement.play();
+
+    console.log("Start streaming!")
+  }
+
+  public stopStreaming() {
+    if (!this.videoStream) return;
+
+    // Stop all tracks on the stream to end streaming
+    this.videoStream.getTracks().forEach((track: { stop: () => any; }) => track.stop());
+    this.videoStream = null; // Clear the stream reference
+
+    console.log("Streaming stopped!");
+}
+
+
+  // Method to start recording
+  public startRecording() {
+    if (!this._renderer) return;
+
+    // Create a stream from the renderer's canvas
+    const stream = this._renderer.domElement.captureStream(60); // 30 FPS
+
+    // Higher quality and bit rate for better video clarity
+    const options = {
+      mimeType: 'video/webm;codecs=vp9',
+      videoBitsPerSecond: 8000000, // 8 Mbps for higher quality
+    };
+
+    this.mediaRecorder = new MediaRecorder(stream, options);
+
+    // Collect data in chunks
+    this.mediaRecorder.ondataavailable = (event) => {
+      if (event.data.size > 0) {
+        this.recordedChunks.push(event.data);
+      }
+    };
+
+    // Start recording
+    this.mediaRecorder.start();
+  }
+
+
+  // Method to stop recording and trigger callback
+  public stopRecording(callback: BlobCallback) {
+    if (!this.mediaRecorder) return;
+
+    // Stop recording and create the video blob
+    this.mediaRecorder.onstop = () => {
+      const recordedBlob = new Blob(this.recordedChunks, { type: 'video/webm' });
+      callback(recordedBlob); // Pass the video blob to the callback
+      this.recordedChunks = []; // Clear chunks for the next recording
+    };
+
+    // Stop the recorder
+    this.mediaRecorder.stop();
+  }
+
   public onMouseClick(event: MouseEvent): boolean {
     if (!this._renderer || !this._camera || !this.model?.vrm) return false;