From bb79a3528a1887d7f2aa7d313f6b46525fd76f1c Mon Sep 17 00:00:00 2001 From: "lixinghua.vendor" Date: Wed, 4 Dec 2024 19:09:33 +0800 Subject: [PATCH] feat: LLM supports audio playback --- .../LLMMultiWheelView/dialogView/index.tsx | 67 +++++++++++-------- .../components/LLMMultiWheelView/index.tsx | 51 +++++++++----- .../src/components/LLMToolView/index.tsx | 6 +- .../components/audioView/index.tsx | 51 ++++++++++++++ .../questionView/components/header/index.tsx | 27 +++++++- .../LLMToolView/questionView/index.tsx | 17 ++++- packages/lb-components/src/constant/index.tsx | 1 + 7 files changed, 171 insertions(+), 49 deletions(-) create mode 100644 packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx diff --git a/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx b/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx index b5c061da3..eb56737af 100644 --- a/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx +++ b/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx @@ -1,4 +1,5 @@ import { getTextControlByConfig, RenderAnswer } from '@/components/LLMToolView/questionView'; +import AudioView from '@/components/LLMToolView/questionView/components/audioView'; import { RenderQuestion } from '@/components/LLMToolView/questionView/components/header'; import ImgView from '@/components/LLMToolView/questionView/components/imgView'; import { ILLMMultiWheelToolConfig } from '@/components/LLMToolView/types'; @@ -18,6 +19,8 @@ interface IDialogViewProps { answerIsImg: boolean; questionIsImg: boolean; LLMConfig?: ILLMMultiWheelToolConfig; + answerIsAudio?: boolean; + questionIsAudio?: boolean; } const DialogView = (props: IDialogViewProps) => { @@ -29,6 +32,8 @@ const DialogView = (props: IDialogViewProps) => { name = '', answerIsImg, questionIsImg, + questionIsAudio, + answerIsAudio, LLMConfig, } = props; const { t } = useTranslation(); @@ -37,6 +42,39 @@ const DialogView = (props: IDialogViewProps) => { const order = index + 1; const showName = name || `${t('Dialog')}${order}`; + const dialogAnswer = () => { + if (answerIsImg) { + return ( + <> + + + + ); + } + if (answerIsAudio) { + return ( + <> + + + + ); + } + return answerList?.map((item: any, index: number) => { + const order = index + 1; + const answer = { ...item, order }; + const isTextControl = getTextControlByConfig(answer, LLMConfig); + return ( +
+ + +
+ ); + }); + }; + return (
{ question={question} dataFormatType={dataFormatType} isImg={questionIsImg} + isAudio={questionIsAudio} />
-
- {answerIsImg ? ( - <> - - - - ) : ( - answerList?.map((item: any, index: number) => { - const order = index + 1; - const answer = { ...item, order }; - const isTextControl = getTextControlByConfig(answer, LLMConfig); - return ( -
- - -
- ); - }) - )} -
+
{dialogAnswer()}
); }; diff --git a/packages/lb-components/src/components/LLMMultiWheelView/index.tsx b/packages/lb-components/src/components/LLMMultiWheelView/index.tsx index 533db51e4..9fd84de48 100644 --- a/packages/lb-components/src/components/LLMMultiWheelView/index.tsx +++ b/packages/lb-components/src/components/LLMMultiWheelView/index.tsx @@ -30,10 +30,20 @@ interface ILLMMultiWheelSourceViewProps { LLMConfig?: ILLMMultiWheelToolConfig; dialogList: any[]; lang?: string; + questionIsAudio?: boolean; + answerIsAudio?: boolean; } export const LLMMultiWheelSourceView: React.FC = (props) => { - const { questionIsImg, answerIsImg, LLMConfig, dialogList, lang = 'cn' } = props; + const { + questionIsImg, + answerIsImg, + questionIsAudio, + answerIsAudio, + LLMConfig, + dialogList, + lang = 'cn', + } = props; const { dataFormatType, setDataFormatType } = useLLMMultiWheelStore(); useEffect(() => { @@ -66,6 +76,8 @@ export const LLMMultiWheelSourceView: React.FC = index={index} questionIsImg={questionIsImg} answerIsImg={answerIsImg} + questionIsAudio={questionIsAudio} + answerIsAudio={answerIsAudio} LLMConfig={LLMConfig} /> ))} @@ -109,6 +121,9 @@ const LLMMultiWheelView: React.FC = (props) => { const [dialogList, setDialogList] = useState([]); const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture; const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture; + const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio; + const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio; + const { t } = useTranslation(); const [, forceRender] = useState(0); @@ -136,23 +151,25 @@ const LLMMultiWheelView: React.FC = (props) => { const newDialogList = textList?.map((item: any, questionIndex: number) => { return { ...item, - question: questionIsImg - ? getInfoFromLLMFile({ - type: 'question', - questionIndex, - llmFile, - }) || item?.question - : item?.question, - answerList: item?.answerList?.map((i: any, answerIndex: number) => { - const mapId = `${item?.id ?? ''}-${i?.id ?? ''}`; - const info = answerIsImg + question: + questionIsImg || questionIsAudio ? getInfoFromLLMFile({ - type: 'answer', + type: 'question', questionIndex, - answerIndex, llmFile, - }) || {} - : {}; + }) || item?.question + : item?.question, + answerList: item?.answerList?.map((i: any, answerIndex: number) => { + const mapId = `${item?.id ?? ''}-${i?.id ?? ''}`; + const info = + answerIsImg || answerIsAudio + ? getInfoFromLLMFile({ + type: 'answer', + questionIndex, + answerIndex, + llmFile, + }) || {} + : {}; return { ...i, answer: i.answer, @@ -164,7 +181,7 @@ const LLMMultiWheelView: React.FC = (props) => { }; }); setDialogList(newDialogList); - }, [newAnswerListMap, questionIsImg, answerIsImg]); + }, [newAnswerListMap, questionIsImg, questionIsAudio, answerIsImg]); useEffect(() => { if (stepList && step) { @@ -192,6 +209,8 @@ const LLMMultiWheelView: React.FC = (props) => { diff --git a/packages/lb-components/src/components/LLMToolView/index.tsx b/packages/lb-components/src/components/LLMToolView/index.tsx index 27d358f96..477bed507 100644 --- a/packages/lb-components/src/components/LLMToolView/index.tsx +++ b/packages/lb-components/src/components/LLMToolView/index.tsx @@ -71,15 +71,17 @@ const LLMToolView: React.FC = (props) => { return; } const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture; + const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio; const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture; + const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio; const qaData = imgList[imgIndex]?.questionList; const llmFile = imgList[imgIndex]?.llmFile; - const titleQuestion = questionIsImg ? llmFile?.question : qaData?.question; + const titleQuestion = questionIsImg || questionIsAudio ? llmFile?.question : qaData?.question; setQuestion(titleQuestion); let list = qaData?.answerList || []; - if (answerIsImg) { + if (answerIsImg || answerIsAudio) { list = llmFile?.answerList || []; } if (LLMConfig?.dataType?.response === ELLMDataType.None) { diff --git a/packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx new file mode 100644 index 000000000..1c6226c72 --- /dev/null +++ b/packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx @@ -0,0 +1,51 @@ +/* + * @file LLM tool audio view + * @Author: lixinghua lixinghua@sensetime.com + * @Date: 2023-11-13 + */ +import React from 'react'; +import { Tag } from 'antd'; +import { prefix } from '@/constant'; +import classNames from 'classnames'; +import { IAnswerList } from '@/components/LLMToolView/types'; + +interface IProps { + hoverKey?: number; + answerList: IAnswerList[]; +} + +const LLMViewCls = `${prefix}-LLMView`; + +const AudioPlayer = React.memo(({ url }: { url?: string }) => ( + +)); + +const AudioView = (props: IProps) => { + const { answerList, hoverKey } = props; + + return ( +
+ {answerList?.length > 0 && + answerList.map((i: IAnswerList, index: number) => { + return ( +
+ {i?.order} +
+ +
+
+ ); + })} +
+ ); +}; + +export default AudioView; diff --git a/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx index 0e21b7e44..7f1987f59 100644 --- a/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx +++ b/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { useTranslation } from 'react-i18next'; import { Resizable } from 're-resizable'; -import { Radio, Image } from 'antd'; +import { Radio, Image, Empty } from 'antd'; import { EDataFormatType, prefix } from '@/constant'; import { FileTextOutlined } from '@ant-design/icons'; import MarkdownView from '@/components/markdownView'; @@ -28,6 +28,7 @@ interface IProps { dataFormatType: EDataFormatType; setDataFormatType: (v: EDataFormatType) => void; isImg?: boolean; + isAudio?: boolean; } const LLMViewCls = `${prefix}-LLMView`; @@ -36,6 +37,7 @@ export const RenderQuestion = ({ question, dataFormatType, isImg, + isAudio, }: { question: | string @@ -48,6 +50,7 @@ export const RenderQuestion = ({ }; dataFormatType: EDataFormatType; isImg?: boolean; + isAudio?: boolean; }) => { const textValue = isString(question) ? question : ''; const ImgFail = i18n.language === 'en' ? ImgFailEn : ImgFailCn; @@ -56,6 +59,19 @@ export const RenderQuestion = ({ const url = isObject(question) ? question?.url : ''; return ; } + + if (isAudio) { + const url = isObject(question) ? question?.url : ''; + if (url) { + return ( + + ); + } + return ; + } + return (
{dataFormatType === EDataFormatType.Markdown ? : textValue} @@ -63,7 +79,7 @@ export const RenderQuestion = ({ ); }; const Header = (props: IProps) => { - const { question, dataFormatType, setDataFormatType, isImg } = props; + const { question, dataFormatType, setDataFormatType, isImg, isAudio } = props; const DEFAULT_HEIGHT = 300; const { t } = useTranslation(); @@ -91,7 +107,12 @@ const Header = (props: IProps) => { />
- +
); diff --git a/packages/lb-components/src/components/LLMToolView/questionView/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/index.tsx index db0f0eb3c..a3b08030a 100644 --- a/packages/lb-components/src/components/LLMToolView/questionView/index.tsx +++ b/packages/lb-components/src/components/LLMToolView/questionView/index.tsx @@ -22,6 +22,7 @@ import DiffMatchPatchComponent from '@/components/diffMatchPatchComponent'; import Header from './components/header'; import ImgView from './components/imgView'; import { isString } from 'lodash'; +import AudioView from './components/audioView'; interface IProps { hoverKey?: number; @@ -95,6 +96,8 @@ const QuestionView: React.FC = (props) => { const [dataFormatType, setDataFormatType] = useState(EDataFormatType.Default); const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture; const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture; + const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio; + const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio; const { t } = useTranslation(); useEffect(() => { @@ -132,6 +135,17 @@ const QuestionView: React.FC = (props) => { )} ); + + const answerContent = () => { + if (answerIsAudio) { + return ; + } + if (answerIsImg) { + return ; + } + return textAnswer; + }; + return (
= (props) => { dataFormatType={dataFormatType} setDataFormatType={setDataFormatType} isImg={questionIsImg} + isAudio={questionIsAudio} />
{t('Answer')} {answerHeaderSlot}
- {answerIsImg ? : textAnswer} + {answerContent()}
); diff --git a/packages/lb-components/src/constant/index.tsx b/packages/lb-components/src/constant/index.tsx index 932cf2968..2cad0e717 100644 --- a/packages/lb-components/src/constant/index.tsx +++ b/packages/lb-components/src/constant/index.tsx @@ -34,4 +34,5 @@ export enum ELLMDataType { Picture = 'picture', Text = 'text', None = 'none', + Audio = 'audio' }