update speech synth

src/clients/discord/index.ts

@@ -38,10 +38,11 @@ import { default as getUuid, default as uuid } from "uuid-by-string";
 import { Agent } from '../../core/agent.ts';
 import { adapter } from "../../core/db.ts";
 import settings from "../../core/settings.ts";
-import { textToSpeech } from "../elevenlabs/index.ts";
 import { AudioMonitor } from "./audioMonitor.ts";
 import { commands } from "./commands.ts";
 import { InterestChannels, ResponseType } from "./types.ts";
+import { SpeechSynthesizer } from "../../services/speechSynthesis.ts";
+import WavEncoder from "wav-encoder";
 
 export const messageHandlerTemplate =
 // `{{actionExamples}}
@@ -100,6 +101,7 @@ export class DiscordClient extends EventEmitter {
   private agent: Agent;
   private bio: string;
   private transcriber: any;
+  speechSynthesizer: SpeechSynthesizer;
 
   constructor(agent: Agent, bio: string) {
     super();
@@ -256,6 +258,25 @@ export class DiscordClient extends EventEmitter {
     }
   }
 
+  async textToSpeech(text: string): Promise<Readable> {
+    if(!this.speechSynthesizer) {
+      this.speechSynthesizer = await SpeechSynthesizer.create("./model.onnx");
+    }
+
+        console.log("Synthesizing speech...");
+    // Synthesize the speech to get a Float32Array of single channel 22050Hz audio data
+    const audio = await this.speechSynthesizer.synthesize("Four score and seven years ago.");
+    console.log("Speech synthesized");
+    // Encode the audio data into a WAV format
+    const { encode } = WavEncoder;
+    const audioData = {
+        sampleRate: 22050,
+        channelData: [audio]
+    };
+    const wavArrayBuffer = encode.sync(audioData);
+    return wavArrayBuffer;
+  }
+
   async speechToText(audioBuffer: Buffer) {
     if (!this.transcriber) {
       console.log("Transcriber not initialized. Initializing now...");
@@ -692,7 +713,7 @@ export class DiscordClient extends EventEmitter {
     if (requestedResponseType == ResponseType.RESPONSE_TEXT) {
       return Readable.from(content);
     } else {
-      return await textToSpeech(content);
+      return await this.textToSpeech(content);
     }
   }
 

src/index.ts

@@ -8,7 +8,7 @@ import { Agent } from './core/agent.ts';
 import fs from "fs";
 import yargs from "yargs";
 import { SpeechSynthesizer } from "./services/speechSynthesis.ts";
-import WavEncoderPkg from "wav-encoder";
+import WavEncoder from "wav-encoder";
 interface Arguments {
     character?: string;
     twitter?: boolean;