Shaw fix characters paths, .ts requirement and missings args

core/package.json

@@ -85,6 +85,7 @@
         "@cliqz/adblocker-playwright": "1.34.0",
         "@coral-xyz/anchor": "^0.30.1",
         "@discordjs/rest": "2.4.0",
+        "@discordjs/opus": "github:discordjs/opus",
         "@discordjs/voice": "0.17.0",
         "@echogarden/espeak-ng-emscripten": "0.3.0",
         "@echogarden/kissfft-wasm": "0.2.0",
@@ -175,6 +176,7 @@
     "trustedDependencies": {
         "onnxruntime-node": "^1.19.2",
         "@discordjs/opus": "github:discordjs/opus",
+        "@discordjs/voice": "0.17.0",
         "sharp": "^0.33.5"
     },
     "peerDependencies": {

core/src/actions/swap.ts

@@ -1,32 +1,30 @@
+import { bs58 } from "@coral-xyz/anchor/dist/cjs/utils/bytes/index.js";
 import {
     Connection,
     Keypair,
     PublicKey,
-    Transaction,
-    VersionedTransaction,
+    VersionedTransaction
 } from "@solana/web3.js";
+import BigNumber from "bignumber.js";
 import fetch from "cross-fetch";
+import { v4 as uuidv4 } from "uuid";
+import { TrustScoreDatabase } from "../adapters/trustScoreDatabase.ts";
+import { composeContext } from "../core/context.ts";
+import { generateObject } from "../core/generation.ts";
+import settings from "../core/settings.ts";
 import {
     ActionExample,
+    HandlerCallback,
     IAgentRuntime,
     Memory,
-    type Action,
-    State,
     ModelClass,
-    HandlerCallback,
+    State,
+    type Action,
 } from "../core/types.ts";
-import { walletProvider } from "../providers/wallet.ts";
-import { composeContext } from "../core/context.ts";
-import { generateObject, generateObjectArray } from "../core/generation.ts";
+import { TokenProvider } from "../providers/token.ts";
+import { TrustScoreProvider } from "../providers/trustScoreProvider.ts";
+import { walletProvider, WalletProvider } from "../providers/wallet.ts";
 import { getTokenDecimals } from "./swapUtils.ts";
-import settings from "../core/settings.ts";
-import { bs58 } from "@coral-xyz/anchor/dist/cjs/utils/bytes/index.js";
-import BigNumber from "bignumber.js";
-import { WalletProvider } from "../providers/wallet.ts";
-import { TrustScoreProvider } from "../providers/trustScoreProvider";
-import { TokenProvider } from "../providers/token";
-import { TrustScoreDatabase } from "../adapters/trustScoreDatabase";
-import { v4 as uuidv4 } from "uuid";
 
 async function swapToken(
     connection: Connection,
@@ -418,6 +416,7 @@ export const executeSwap: Action = {
                     is_simulation: false,
                 };
                 await trustScoreDatabase.createTradePerformance(
+                    runtime,
                     response.outputTokenCA,
                     recommender.id,
                     tradeData
@@ -448,6 +447,7 @@ export const executeSwap: Action = {
                 };
                 const sellTimeStamp = new Date().getTime().toString();
                 await trustScoreDatabase.updateSellDetails(
+                    runtime,
                     response.inputTokenCA,
                     recommender.id,
                     sellTimeStamp,

core/src/cli/index.ts

@@ -66,7 +66,17 @@ export function parseArguments(): Arguments {
 }
 
 export function loadCharacters(charactersArg: string): Character[] {
-    const characterPaths = charactersArg?.split(",").map((path) => path.trim());
+    let characterPaths = charactersArg
+        ?.split(",")
+        .map((path) => path.trim())
+        .map((path) => {
+            if (path.startsWith("./characters")) {
+                return `../characters/${path}`;
+            }
+            return path;
+        });
+
+
     const loadedCharacters = [];
 
     if (characterPaths?.length > 0) {

core/src/clients/chat/index.ts

@@ -0,0 +1,274 @@
+import bodyParser from "body-parser";
+import express from "express";
+import { composeContext } from "../../core/context.ts";
+import { AgentRuntime } from "../../core/runtime.ts";
+import { Content, Memory, ModelClass, State } from "../../core/types.ts";
+import { stringToUuid } from "../../core/uuid.ts";
+import cors from "cors";
+import { messageCompletionFooter } from "../../core/parsing.ts";
+import multer, { File } from "multer";
+import { Request as ExpressRequest } from "express";
+import { generateMessageResponse } from "../../core/generation.ts";
+import {
+    generateCaption,
+    generateImage,
+} from "../../actions/imageGenerationUtils.ts";
+
+const upload = multer({ storage: multer.memoryStorage() });
+
+export const messageHandlerTemplate =
+    // {{goals}}
+    `# Action Examples
+{{actionExamples}}
+(Action examples are for reference only. Do not use the information from them in your response.)
+
+# Task: Generate dialog and actions for the character {{agentName}}.
+About {{agentName}}:
+{{bio}}
+{{lore}}
+
+{{providers}}
+
+{{attachments}}
+
+# Capabilities
+Note that {{agentName}} is capable of reading/seeing/hearing various forms of media, including images, videos, audio, plaintext and PDFs. Recent attachments have been included above under the "Attachments" section.
+
+{{messageDirections}}
+
+{{recentMessages}}
+
+{{actions}}
+
+# Instructions: Write the next message for {{agentName}}. Ignore "action".
+` + messageCompletionFooter;
+
+export interface SimliClientConfig {
+    apiKey: string;
+    faceID: string;
+    handleSilence: boolean;
+    videoRef: any;
+    audioRef: any;
+}
+class DirectClient {
+    private app: express.Application;
+    private agents: Map<string, AgentRuntime>;
+
+    constructor() {
+        this.app = express();
+        this.app.use(cors());
+        this.agents = new Map();
+
+        this.app.use(bodyParser.json());
+        this.app.use(bodyParser.urlencoded({ extended: true }));
+
+        // Define an interface that extends the Express Request interface
+        interface CustomRequest extends ExpressRequest {
+            file: File;
+        }
+
+        // Update the route handler to use CustomRequest instead of express.Request
+        this.app.post(
+            "/:agentId/whisper",
+            upload.single("file"),
+            async (req: CustomRequest, res: express.Response) => {
+                const audioFile = req.file; // Access the uploaded file using req.file
+                const agentId = req.params.agentId;
+
+                if (!audioFile) {
+                    res.status(400).send("No audio file provided");
+                    return;
+                }
+
+                let runtime = this.agents.get(agentId);
+
+                // if runtime is null, look for runtime with the same name
+                if (!runtime) {
+                    runtime = Array.from(this.agents.values()).find(
+                        (a) =>
+                            a.character.name.toLowerCase() ===
+                            agentId.toLowerCase()
+                    );
+                }
+
+                if (!runtime) {
+                    res.status(404).send("Agent not found");
+                    return;
+                }
+
+                const formData = new FormData();
+                const audioBlob = new Blob([audioFile.buffer], {
+                    type: audioFile.mimetype,
+                });
+                formData.append("file", audioBlob, audioFile.originalname);
+                formData.append("model", "whisper-1");
+
+                const response = await fetch(
+                    "https://api.openai.com/v1/audio/transcriptions",
+                    {
+                        method: "POST",
+                        headers: {
+                            Authorization: `Bearer ${runtime.token}`,
+                        },
+                        body: formData,
+                    }
+                );
+
+                const data = await response.json();
+                res.json(data);
+            }
+        );
+
+        this.app.post(
+            "/:agentId/message",
+            async (req: express.Request, res: express.Response) => {
+                const agentId = req.params.agentId;
+                const roomId = stringToUuid(
+                    req.body.roomId ?? "default-room-" + agentId
+                );
+                const userId = stringToUuid(req.body.userId ?? "user");
+
+                let runtime = this.agents.get(agentId);
+
+                // if runtime is null, look for runtime with the same name
+                if (!runtime) {
+                    runtime = Array.from(this.agents.values()).find(
+                        (a) =>
+                            a.character.name.toLowerCase() ===
+                            agentId.toLowerCase()
+                    );
+                }
+
+                if (!runtime) {
+                    res.status(404).send("Agent not found");
+                    return;
+                }
+
+                await runtime.ensureConnection(
+                    userId,
+                    roomId,
+                    req.body.userName,
+                    req.body.name,
+                    "direct"
+                );
+
+                const text = req.body.text;
+                const messageId = stringToUuid(Date.now().toString());
+
+                const content: Content = {
+                    text,
+                    attachments: [],
+                    source: "direct",
+                    inReplyTo: undefined,
+                };
+
+                const userMessage = { content, userId, roomId, agentId: runtime.agentId };
+
+                const memory: Memory = {
+                    id: messageId,
+                    agentId: runtime.agentId,
+                    userId,
+                    roomId,
+                    content,
+                    createdAt: Date.now(),
+                };
+
+                await runtime.messageManager.createMemory(memory);
+
+                const state = (await runtime.composeState(userMessage, {
+                    agentName: runtime.character.name,
+                })) as State;
+
+                const context = composeContext({
+                    state,
+                    template: messageHandlerTemplate,
+                });
+
+                const response = await generateMessageResponse({
+                    runtime: runtime,
+                    context,
+                    modelClass: ModelClass.SMALL,
+                });
+
+                // save response to memory
+                const responseMessage = {
+                    ...userMessage,
+                    userId: runtime.agentId,
+                    content: response,
+                };
+
+                await runtime.messageManager.createMemory(responseMessage);
+
+                if (!response) {
+                    res.status(500).send(
+                        "No response from generateMessageResponse"
+                    );
+                    return;
+                }
+
+                let message = null as Content | null;
+
+                const result = await runtime.processActions(
+                    memory,
+                    [responseMessage],
+                    state,
+                    async (newMessages) => {
+                        message = newMessages;
+                        return [memory];
+                    }
+                )
+
+                if (message) {
+                    res.json([message, response]);
+                } else {
+                    res.json([response]);
+                }
+
+            }
+        );
+
+        this.app.post(
+            "/:agentId/image",
+            async (req: express.Request, res: express.Response) => {
+                const agentId = req.params.agentId;
+                const agent = this.agents.get(agentId);
+                if (!agent) {
+                    res.status(404).send("Agent not found");
+                    return;
+                }
+
+                const images = await generateImage({ ...req.body }, agent);
+                const imagesRes: { image: string; caption: string }[] = [];
+                if (images.data && images.data.length > 0) {
+                    for (let i = 0; i < images.data.length; i++) {
+                        const caption = await generateCaption(
+                            { imageUrl: images.data[i] },
+                            agent
+                        );
+                        imagesRes.push({
+                            image: images.data[i],
+                            caption: caption.title,
+                        });
+                    }
+                }
+                res.json({ images: imagesRes });
+            }
+        );
+    }
+
+    public registerAgent(runtime: AgentRuntime) {
+        this.agents.set(runtime.agentId, runtime);
+    }
+
+    public unregisterAgent(runtime: AgentRuntime) {
+        this.agents.delete(runtime.agentId);
+    }
+
+    public start(port: number) {
+        this.app.listen(port, () => {
+            console.log(`Server running at http://localhost:${port}/`);
+        });
+    }
+}
+
+export { DirectClient };