From 0cbc096b64ffc5083e6d861b6c13fedba12fed6d Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Sun, 5 Jan 2025 16:56:22 +0800 Subject: [PATCH] perf: async read file --- .../zh-cn/docs/development/upgrading/4818.md | 3 +- packages/plugins/register.ts | 32 ++++++++++--------- packages/service/common/file/read/utils.ts | 3 +- .../service/worker/readFile/parseOffice.ts | 16 ++++++---- packages/templates/register.ts | 24 +++++++------- .../app/src/service/common/system/index.ts | 32 ++++++++++--------- projects/app/src/service/core/app/plugin.ts | 4 ++- 7 files changed, 61 insertions(+), 53 deletions(-) diff --git a/docSite/content/zh-cn/docs/development/upgrading/4818.md b/docSite/content/zh-cn/docs/development/upgrading/4818.md index 55dc1ddc29b9..b87c83fd409c 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4818.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4818.md @@ -15,4 +15,5 @@ weight: 806 4. 优化 - Mongo 全文索引表分离。 5. 优化 - 知识库检索查询语句合并，同时减少查库数量。 6. 优化 - 文件编码检测，减少 CSV 文件乱码概率。 -7. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。 \ No newline at end of file +7. 优化 - 异步读取文件内容，减少进程阻塞。 +8. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。 \ No newline at end of file diff --git a/packages/plugins/register.ts b/packages/plugins/register.ts index 2509422d2f7b..9989fbc78761 100644 --- a/packages/plugins/register.ts +++ b/packages/plugins/register.ts @@ -35,24 +35,26 @@ export const list = [...staticPluginList, ...packagePluginList]; /* Get plugins */ export const getCommunityPlugins = () => { - return list.map((name) => { - const config = require(`./src/${name}/template.json`); + return Promise.all( + list.map>(async (name) => { + const config = (await import(`./src/${name}/template.json`))?.default; - const isFolder = list.find((item) => item.startsWith(`${name}/`)); + const isFolder = list.find((item) => item.startsWith(`${name}/`)); - const parentIdList = name.split('/').slice(0, -1); - const parentId = - parentIdList.length > 0 ? `${PluginSourceEnum.community}-${parentIdList.join('/')}` : null; + const parentIdList = name.split('/').slice(0, -1); + const parentId = + parentIdList.length > 0 ? `${PluginSourceEnum.community}-${parentIdList.join('/')}` : null; - return { - ...config, - id: `${PluginSourceEnum.community}-${name}`, - isFolder, - parentId, - isActive: true, - isOfficial: true - }; - }); + return { + ...config, + id: `${PluginSourceEnum.community}-${name}`, + isFolder, + parentId, + isActive: true, + isOfficial: true + }; + }) + ); }; export const getSystemPluginTemplates = () => { diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index ac3afe1a6f2c..7f41f6c47d30 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -7,7 +7,6 @@ import type { ReadFileResponse } from '../../../worker/readFile/type'; import axios from 'axios'; import { addLog } from '../../system/log'; import { batchRun } from '@fastgpt/global/common/fn/utils'; -import { addHours } from 'date-fns'; import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown'; export type readRawTextByLocalFileParams = { @@ -21,7 +20,7 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam const extension = path?.split('.')?.pop()?.toLowerCase() || ''; - const buffer = fs.readFileSync(path); + const buffer = await fs.promises.readFile(path); const { rawText } = await readRawContentByFileBuffer({ extension, diff --git a/packages/service/worker/readFile/parseOffice.ts b/packages/service/worker/readFile/parseOffice.ts index 05dabf9ecc3e..177f5e2cdeeb 100644 --- a/packages/service/worker/readFile/parseOffice.ts +++ b/packages/service/worker/readFile/parseOffice.ts @@ -44,13 +44,15 @@ const parsePowerPoint = async ({ } // Returning an array of all the xml contents read using fs.readFileSync - const xmlContentArray = files.map((file) => { - try { - return fs.readFileSync(`${decompressPath}/${file.path}`, encoding); - } catch (err) { - return fs.readFileSync(`${decompressPath}/${file.path}`, 'utf-8'); - } - }); + const xmlContentArray = await Promise.all( + files.map((file) => { + try { + return fs.promises.readFile(`${decompressPath}/${file.path}`, encoding); + } catch (err) { + return fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8'); + } + }) + ); let responseArr: string[] = []; diff --git a/packages/templates/register.ts b/packages/templates/register.ts index 4a18d4d96507..fae3334ea854 100644 --- a/packages/templates/register.ts +++ b/packages/templates/register.ts @@ -12,24 +12,24 @@ const getTemplateNameList = () => { return fs.readdirSync(templatesPath) as string[]; }; -const getFileTemplates = (): AppTemplateSchemaType[] => { +const getFileTemplates = async (): Promise => { const templateNames = getTemplateNameList(); - const appMarketTemplates = templateNames.map((name) => { - const fileContent = require(`./src/${name}/template.json`); + return Promise.all( + templateNames.map>(async (name) => { + const fileContent = (await import(`./src/${name}/template.json`))?.default; - return { - ...fileContent, - templateId: `${PluginSourceEnum.community}-${name}`, - isActive: true - }; - }); - - return appMarketTemplates; + return { + ...fileContent, + templateId: `${PluginSourceEnum.community}-${name}`, + isActive: true + }; + }) + ); }; const getAppTemplates = async () => { - const communityTemplates = getFileTemplates(); + const communityTemplates = await getFileTemplates(); const dbTemplates = await MongoAppTemplate.find(); diff --git a/projects/app/src/service/common/system/index.ts b/projects/app/src/service/common/system/index.ts index dc017ec01e69..2583dc5ac396 100644 --- a/projects/app/src/service/common/system/index.ts +++ b/projects/app/src/service/common/system/index.ts @@ -1,5 +1,5 @@ import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent'; -import { existsSync, readdirSync, readFileSync } from 'fs'; +import fs, { existsSync, readdirSync } from 'fs'; import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d'; import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d'; import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants'; @@ -13,7 +13,7 @@ import { defaultGroup, defaultTemplateTypes } from '@fastgpt/web/core/workflow/c import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema'; import { MongoTemplateTypes } from '@fastgpt/service/core/app/templates/templateTypeSchema'; -export const readConfigData = (name: string) => { +export const readConfigData = async (name: string) => { const splitName = name.split('.'); const devName = `${splitName[0]}.local.${splitName[1]}`; @@ -30,7 +30,7 @@ export const readConfigData = (name: string) => { return `/app/data/${name}`; })(); - const content = readFileSync(filename, 'utf-8'); + const content = await fs.promises.readFile(filename, 'utf-8'); return content; }; @@ -120,13 +120,13 @@ export async function initSystemConfig() { }); } -function getSystemVersion() { +async function getSystemVersion() { if (global.systemVersion) return; try { if (process.env.NODE_ENV === 'development') { global.systemVersion = process.env.npm_package_version || '0.0.0'; } else { - const packageJson = json5.parse(readFileSync('/app/package.json', 'utf-8')); + const packageJson = json5.parse(await fs.promises.readFile('/app/package.json', 'utf-8')); global.systemVersion = packageJson?.version; } @@ -138,7 +138,7 @@ function getSystemVersion() { } } -function getSystemPlugin() { +async function getSystemPlugin() { if (global.communityPlugins && global.communityPlugins.length > 0) return; const basePath = @@ -149,15 +149,17 @@ function getSystemPlugin() { const filterFiles = files.filter((item) => item.endsWith('.json')); // read json file - const fileTemplates = filterFiles.map((filename) => { - const content = readFileSync(`${basePath}/${filename}`, 'utf-8'); - return { - ...json5.parse(content), - originCost: 0, - currentCost: 0, - id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}` - }; - }); + const fileTemplates = await Promise.all( + filterFiles.map>(async (filename) => { + const content = await fs.promises.readFile(`${basePath}/${filename}`, 'utf-8'); + return { + ...json5.parse(content), + originCost: 0, + currentCost: 0, + id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}` + }; + }) + ); fileTemplates.sort((a, b) => (b.weight || 0) - (a.weight || 0)); diff --git a/projects/app/src/service/core/app/plugin.ts b/projects/app/src/service/core/app/plugin.ts index 3147803ede32..4d98c0baf891 100644 --- a/projects/app/src/service/core/app/plugin.ts +++ b/projects/app/src/service/core/app/plugin.ts @@ -21,7 +21,9 @@ export const getSystemPlugins = async (refresh = false) => { global.systemPlugins = []; } - global.systemPlugins = FastGPTProUrl ? await getCommercialPlugins() : getCommunityPlugins(); + global.systemPlugins = FastGPTProUrl + ? await getCommercialPlugins() + : await getCommunityPlugins(); addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`);