From 8cc331de80a546d0bc584d3b25a3e5203a68d741 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Tue, 21 Nov 2023 00:16:11 +0530 Subject: [PATCH 1/5] Refactor DocxQueueController to use DialoqbaseDocxLoader --- server/src/loader/docx.ts | 22 +++++++++++++++++++ .../src/queue/controllers/docx.controller.ts | 11 +++++----- 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 server/src/loader/docx.ts diff --git a/server/src/loader/docx.ts b/server/src/loader/docx.ts new file mode 100644 index 00000000..c481f403 --- /dev/null +++ b/server/src/loader/docx.ts @@ -0,0 +1,22 @@ +import { BufferLoader } from "langchain/document_loaders/fs/buffer"; +import { Document } from "langchain/document"; +import * as mammoth from "mammoth"; + +export class DialoqbaseDocxLoader extends BufferLoader { + constructor(filePathOrBlob: string | Blob) { + super(filePathOrBlob); + } + + public async parse( + raw: Buffer, + metadata: Document["metadata"] + ): Promise { + const data = await mammoth.extractRawText({ buffer: raw }); + const text = data.value; + const meta = { source: this.filePathOrBlob }; + if (text) { + return [new Document({ pageContent: text, metadata: meta })]; + } + return []; + } +} diff --git a/server/src/queue/controllers/docx.controller.ts b/server/src/queue/controllers/docx.controller.ts index c1303544..e0f9bcf7 100644 --- a/server/src/queue/controllers/docx.controller.ts +++ b/server/src/queue/controllers/docx.controller.ts @@ -1,16 +1,15 @@ -import { DocxLoader } from "langchain/document_loaders/fs/docx"; +// import { DocxLoader } from "langchain/document_loaders/fs/docx"; import { QSource } from "../type"; import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; import { DialoqbaseVectorStore } from "../../utils/store"; import { embeddings } from "../../utils/embeddings"; +import { DialoqbaseDocxLoader } from "../../loader/docx"; -export const DocxQueueController = async ( - source: QSource, -) => { +export const DocxQueueController = async (source: QSource) => { console.log("loading docx"); const location = source.location!; - const loader = new DocxLoader(location); + const loader = new DialoqbaseDocxLoader(location); const docs = await loader.load(); const textSplitter = new RecursiveCharacterTextSplitter({ @@ -25,6 +24,6 @@ export const DocxQueueController = async ( { botId: source.botId, sourceId: source.id, - }, + } ); }; From 6893cd702331ec3fe04325eb7b69fbfbdbda472c Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Wed, 22 Nov 2023 00:24:28 +0530 Subject: [PATCH 2/5] Update package versions --- app/ui/package.json | 2 +- package.json | 2 +- server/package.json | 12 ++++++++---- server/yarn.lock | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/ui/package.json b/app/ui/package.json index 5962ce98..daabe64e 100644 --- a/app/ui/package.json +++ b/app/ui/package.json @@ -1,7 +1,7 @@ { "name": "app", "private": true, - "version": "1.4.2", + "version": "1.4.3", "type": "module", "scripts": { "dev": "vite", diff --git a/package.json b/package.json index a0153be0..90e58ef0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dialoqbase", - "version": "1.4.2", + "version": "1.4.3", "description": "Create chatbots with ease", "scripts": { "ui:dev": "pnpm run --filter ui dev", diff --git a/server/package.json b/server/package.json index ed32588e..8b957f2d 100644 --- a/server/package.json +++ b/server/package.json @@ -9,7 +9,8 @@ "scripts": { "test": "npm run build:ts && tsc -p test/tsconfig.json && tap --ts \"test/**/*.test.ts\"", "start:test": "npm run build:ts && fastify start dist/app.js", - "start": "npx prisma migrate deploy && fastify start app.js", + "start": "npx prisma migrate deploy && npx prisma db seed && fastify start app.js", + "seed": "npx prisma db seed", "build:ts": "tsc && copyfiles -u 1 \"src/**/*.js \" dist", "build": "tsc && copyfiles -u 1 \"src/**/*.js \" dist", "db": "prisma generate", @@ -19,7 +20,7 @@ "dev:start": "fastify start --ignore-watch=uploads,.ts$ -w -P dist/app.js" }, "keywords": [], - "author": "", + "author": "", "license": "MIT", "dependencies": { "@fastify/autoload": "^5.0.0", @@ -71,6 +72,7 @@ "sitemapper": "^3.2.6", "unique-names-generator": "^4.7.1", "wavefile": "^11.0.0", + "ts-node": "^10.9.1", "ytdl-core": "^4.11.5" }, "devDependencies": { @@ -81,7 +83,9 @@ "fastify-tsconfig": "^1.0.1", "prisma": "5", "tap": "^16.1.0", - "ts-node": "^10.4.0", "typescript": "^4.5.4" + }, + "prisma": { + "seed": "ts-node prisma/seed.ts" } -} +} \ No newline at end of file diff --git a/server/yarn.lock b/server/yarn.lock index f68c9e28..049a78d2 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -6315,7 +6315,7 @@ ts-mixer@^6.0.3: resolved "https://registry.yarnpkg.com/ts-mixer/-/ts-mixer-6.0.3.tgz#69bd50f406ff39daa369885b16c77a6194c7cae6" integrity sha512-k43M7uCG1AkTyxgnmI5MPwKoUvS/bRvLvUb7+Pgpdlmok8AoqmUaZxUUw8zKM5B1lqZrt41GjYgnvAi0fppqgQ== -ts-node@^10.4.0: +ts-node@^10.9.1: version "10.9.1" resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== From 87c62f703fc1936591bbab5e180cb29561ff82a1 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Wed, 22 Nov 2023 00:24:41 +0530 Subject: [PATCH 3/5] New Seed Script --- server/prisma/seed.ts | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 server/prisma/seed.ts diff --git a/server/prisma/seed.ts b/server/prisma/seed.ts new file mode 100644 index 00000000..533372da --- /dev/null +++ b/server/prisma/seed.ts @@ -0,0 +1,50 @@ +import { PrismaClient } from "@prisma/client"; +const prisma = new PrismaClient(); + +const MODELS = [ + { + name: "Claude 2.1 (Anthropic)", + model_id: "claude-2.1-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, + { + name: "Claude Instant 1.2 (Anthropic)", + model_id: "claude-instant-1.2-dbase", + model_type: "chat", + model_provider: "Anthropic", + stream_available: true, + local_model: false, + config: "{}", + }, +]; + +const newModels = async () => { + console.log("Seeding new models..."); + for (const model of MODELS) { + await prisma.dialoqbaseModels.upsert({ + where: { + model_id: model.model_id, + }, + update: {}, + create: model, + }); + } +}; + +const main = async () => { + await newModels(); +}; + +main() + .then(async () => { + await prisma.$disconnect(); + }) + .catch(async (e) => { + console.error(e); + await prisma.$disconnect(); + process.exit(1); + }); From 0214615bbf4321d409083a3b0cae7710a9689284 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Wed, 22 Nov 2023 00:27:10 +0530 Subject: [PATCH 4/5] Update push settings in next.yml --- .github/workflows/next.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/next.yml b/.github/workflows/next.yml index a4f4fdcb..c5e63788 100644 --- a/.github/workflows/next.yml +++ b/.github/workflows/next.yml @@ -24,7 +24,4 @@ jobs: with: tags: | n4z3m/dialoqbase-next:latest - push: true - platforms: linux/amd64,linux/arm64 - cache-from: type=registry,ref=n4z3m/dialoqbase-next:latest - cache-to: type=inline \ No newline at end of file + push: true \ No newline at end of file From 000b8894bb54f26c822af581c792cd405fe00ace Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Thu, 23 Nov 2023 18:11:14 +0530 Subject: [PATCH 5/5] Refactor form component in Playground Update message rendering in PlaygroundMessage --- app/ui/src/components/Bot/Playground/Form.tsx | 113 +++++++++--------- .../src/components/Bot/Playground/Message.tsx | 6 +- 2 files changed, 58 insertions(+), 61 deletions(-) diff --git a/app/ui/src/components/Bot/Playground/Form.tsx b/app/ui/src/components/Bot/Playground/Form.tsx index c6757a63..97dd5004 100644 --- a/app/ui/src/components/Bot/Playground/Form.tsx +++ b/app/ui/src/components/Bot/Playground/Form.tsx @@ -110,13 +110,16 @@ export const PlaygroundgForm = () => { })} className="shrink-0 flex-grow flex items-center " > -
+