Merge pull request #615 from lixinghua123/3.8.5-lxh

feat: LLM supports audio playback
open-mmlab · Dec 5, 2024 · bf285e5 · bf285e5
2 parents cc1e00f + bb79a35
commit bf285e5
Show file tree

Hide file tree

Showing 7 changed files with 171 additions and 49 deletions.
diff --git a/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx b/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx
@@ -1,4 +1,5 @@
 import { getTextControlByConfig, RenderAnswer } from '@/components/LLMToolView/questionView';
+import AudioView from '@/components/LLMToolView/questionView/components/audioView';
 import { RenderQuestion } from '@/components/LLMToolView/questionView/components/header';
 import ImgView from '@/components/LLMToolView/questionView/components/imgView';
 import { ILLMMultiWheelToolConfig } from '@/components/LLMToolView/types';
@@ -18,6 +19,8 @@ interface IDialogViewProps {
   answerIsImg: boolean;
   questionIsImg: boolean;
   LLMConfig?: ILLMMultiWheelToolConfig;
+  answerIsAudio?: boolean;
+  questionIsAudio?: boolean;
 }
 
 const DialogView = (props: IDialogViewProps) => {
@@ -29,6 +32,8 @@ const DialogView = (props: IDialogViewProps) => {
     name = '',
     answerIsImg,
     questionIsImg,
+    questionIsAudio,
+    answerIsAudio,
     LLMConfig,
   } = props;
   const { t } = useTranslation();
@@ -37,6 +42,39 @@ const DialogView = (props: IDialogViewProps) => {
   const order = index + 1;
   const showName = name || `${t('Dialog')}${order}`;
 
+  const dialogAnswer = () => {
+    if (answerIsImg) {
+      return (
+        <>
+          <Button type='primary'>{t('Answer')}</Button>
+          <ImgView answerList={answerList} />
+        </>
+      );
+    }
+    if (answerIsAudio) {
+      return (
+        <>
+          <Button type='primary'>{t('Answer')}</Button>
+          <AudioView answerList={answerList} />
+        </>
+      );
+    }
+    return answerList?.map((item: any, index: number) => {
+      const order = index + 1;
+      const answer = { ...item, order };
+      const isTextControl = getTextControlByConfig(answer, LLMConfig);
+      return (
+        <div key={index}>
+          <Button type='primary'>
+            {t('Answer')}
+            {order}
+          </Button>
+          <RenderAnswer i={answer} isTextControl={isTextControl} dataFormatType={dataFormatType} />
+        </div>
+      );
+    });
+  };
+
   return (
     <div
       key={id}
@@ -63,36 +101,11 @@ const DialogView = (props: IDialogViewProps) => {
             question={question}
             dataFormatType={dataFormatType}
             isImg={questionIsImg}
+            isAudio={questionIsAudio}
           />
         </div>
       </div>
-      <div className={`dialog-answer`}>
-        {answerIsImg ? (
-          <>
-            <Button type='primary'>{t('Answer')}</Button>
-            <ImgView answerList={answerList} />
-          </>
-        ) : (
-          answerList?.map((item: any, index: number) => {
-            const order = index + 1;
-            const answer = { ...item, order };
-            const isTextControl = getTextControlByConfig(answer, LLMConfig);
-            return (
-              <div key={index}>
-                <Button type='primary'>
-                  {t('Answer')}
-                  {order}
-                </Button>
-                <RenderAnswer
-                  i={answer}
-                  isTextControl={isTextControl}
-                  dataFormatType={dataFormatType}
-                />
-              </div>
-            );
-          })
-        )}
-      </div>
+      <div className={`dialog-answer`}>{dialogAnswer()}</div>
     </div>
   );
 };

diff --git a/packages/lb-components/src/components/LLMMultiWheelView/index.tsx b/packages/lb-components/src/components/LLMMultiWheelView/index.tsx
@@ -30,10 +30,20 @@ interface ILLMMultiWheelSourceViewProps {
   LLMConfig?: ILLMMultiWheelToolConfig;
   dialogList: any[];
   lang?: string;
+  questionIsAudio?: boolean;
+  answerIsAudio?: boolean;
 }
 
 export const LLMMultiWheelSourceView: React.FC<ILLMMultiWheelSourceViewProps> = (props) => {
-  const { questionIsImg, answerIsImg, LLMConfig, dialogList, lang = 'cn' } = props;
+  const {
+    questionIsImg,
+    answerIsImg,
+    questionIsAudio,
+    answerIsAudio,
+    LLMConfig,
+    dialogList,
+    lang = 'cn',
+  } = props;
   const { dataFormatType, setDataFormatType } = useLLMMultiWheelStore();
 
   useEffect(() => {
@@ -66,6 +76,8 @@ export const LLMMultiWheelSourceView: React.FC<ILLMMultiWheelSourceViewProps> =
               index={index}
               questionIsImg={questionIsImg}
               answerIsImg={answerIsImg}
+              questionIsAudio={questionIsAudio}
+              answerIsAudio={answerIsAudio}
               LLMConfig={LLMConfig}
             />
           ))}
@@ -109,6 +121,9 @@ const LLMMultiWheelView: React.FC<IProps> = (props) => {
   const [dialogList, setDialogList] = useState<any[]>([]);
   const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture;
   const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture;
+  const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio;
+  const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio;
+
   const { t } = useTranslation();
   const [, forceRender] = useState(0);
 
@@ -136,23 +151,25 @@ const LLMMultiWheelView: React.FC<IProps> = (props) => {
     const newDialogList = textList?.map((item: any, questionIndex: number) => {
       return {
         ...item,
-        question: questionIsImg
-          ? getInfoFromLLMFile({
-              type: 'question',
-              questionIndex,
-              llmFile,
-            }) || item?.question
-          : item?.question,
-        answerList: item?.answerList?.map((i: any, answerIndex: number) => {
-          const mapId = `${item?.id ?? ''}-${i?.id ?? ''}`;
-          const info = answerIsImg
+        question:
+          questionIsImg || questionIsAudio
             ? getInfoFromLLMFile({
-                type: 'answer',
+                type: 'question',
                 questionIndex,
-                answerIndex,
                 llmFile,
-              }) || {}
-            : {};
+              }) || item?.question
+            : item?.question,
+        answerList: item?.answerList?.map((i: any, answerIndex: number) => {
+          const mapId = `${item?.id ?? ''}-${i?.id ?? ''}`;
+          const info =
+            answerIsImg || answerIsAudio
+              ? getInfoFromLLMFile({
+                  type: 'answer',
+                  questionIndex,
+                  answerIndex,
+                  llmFile,
+                }) || {}
+              : {};
           return {
             ...i,
             answer: i.answer,
@@ -164,7 +181,7 @@ const LLMMultiWheelView: React.FC<IProps> = (props) => {
       };
     });
     setDialogList(newDialogList);
-  }, [newAnswerListMap, questionIsImg, answerIsImg]);
+  }, [newAnswerListMap, questionIsImg, questionIsAudio, answerIsImg]);
 
   useEffect(() => {
     if (stepList && step) {
@@ -192,6 +209,8 @@ const LLMMultiWheelView: React.FC<IProps> = (props) => {
       <LLMMultiWheelSourceView
         questionIsImg={questionIsImg}
         answerIsImg={answerIsImg}
+        questionIsAudio={questionIsAudio}
+        answerIsAudio={answerIsAudio}
         LLMConfig={LLMConfig}
         dialogList={dialogList}
       />

diff --git a/packages/lb-components/src/components/LLMToolView/index.tsx b/packages/lb-components/src/components/LLMToolView/index.tsx
@@ -71,15 +71,17 @@ const LLMToolView: React.FC<IProps> = (props) => {
       return;
     }
     const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture;
+    const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio;
     const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture;
+    const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio;
 
     const qaData = imgList[imgIndex]?.questionList;
 
     const llmFile = imgList[imgIndex]?.llmFile;
-    const titleQuestion = questionIsImg ? llmFile?.question : qaData?.question;
+    const titleQuestion = questionIsImg || questionIsAudio ? llmFile?.question : qaData?.question;
     setQuestion(titleQuestion);
     let list = qaData?.answerList || [];
-    if (answerIsImg) {
+    if (answerIsImg || answerIsAudio) {
       list = llmFile?.answerList || [];
     }
     if (LLMConfig?.dataType?.response === ELLMDataType.None) {

diff --git a/...ages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx b/...ages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx
@@ -0,0 +1,51 @@
+/*
+ * @file LLM tool audio view
+ * @Author: lixinghua [email protected]
+ * @Date: 2023-11-13
+ */
+import React from 'react';
+import { Tag } from 'antd';
+import { prefix } from '@/constant';
+import classNames from 'classnames';
+import { IAnswerList } from '@/components/LLMToolView/types';
+
+interface IProps {
+  hoverKey?: number;
+  answerList: IAnswerList[];
+}
+
+const LLMViewCls = `${prefix}-LLMView`;
+
+const AudioPlayer = React.memo(({ url }: { url?: string }) => (
+  <audio controls>
+    <source src={url} type='audio/mpeg' />
+  </audio>
+));
+
+const AudioView = (props: IProps) => {
+  const { answerList, hoverKey } = props;
+
+  return (
+    <div>
+      {answerList?.length > 0 &&
+        answerList.map((i: IAnswerList, index: number) => {
+          return (
+            <div
+              key={index}
+              className={classNames({
+                [`${LLMViewCls}__content`]: true,
+                [`${LLMViewCls}__contentActive`]: hoverKey === i?.order,
+              })}
+            >
+              <Tag className={`${LLMViewCls}-tag`}>{i?.order}</Tag>
+              <div>
+                <AudioPlayer key={i?.url} url={i?.url} />
+              </div>
+            </div>
+          );
+        })}
+    </div>
+  );
+};
+
+export default AudioView;
diff --git a/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx
@@ -6,7 +6,7 @@
 import React from 'react';
 import { useTranslation } from 'react-i18next';
 import { Resizable } from 're-resizable';
-import { Radio, Image } from 'antd';
+import { Radio, Image, Empty } from 'antd';
 import { EDataFormatType, prefix } from '@/constant';
 import { FileTextOutlined } from '@ant-design/icons';
 import MarkdownView from '@/components/markdownView';
@@ -28,6 +28,7 @@ interface IProps {
   dataFormatType: EDataFormatType;
   setDataFormatType: (v: EDataFormatType) => void;
   isImg?: boolean;
+  isAudio?: boolean;
 }
 
 const LLMViewCls = `${prefix}-LLMView`;
@@ -36,6 +37,7 @@ export const RenderQuestion = ({
   question,
   dataFormatType,
   isImg,
+  isAudio,
 }: {
   question:
     | string
@@ -48,6 +50,7 @@ export const RenderQuestion = ({
       };
   dataFormatType: EDataFormatType;
   isImg?: boolean;
+  isAudio?: boolean;
 }) => {
   const textValue = isString(question) ? question : '';
   const ImgFail = i18n.language === 'en' ? ImgFailEn : ImgFailCn;
@@ -56,14 +59,27 @@ export const RenderQuestion = ({
     const url = isObject(question) ? question?.url : '';
     return <Image src={url || ImgFail} fallback={ImgFail} />;
   }
+
+  if (isAudio) {
+    const url = isObject(question) ? question?.url : '';
+    if (url) {
+      return (
+        <audio controls>
+          <source src={url} type='audio/mpeg' />
+        </audio>
+      );
+    }
+    return <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} />;
+  }
+
   return (
     <div style={{ whiteSpace: 'pre-wrap' }}>
       {dataFormatType === EDataFormatType.Markdown ? <MarkdownView value={textValue} /> : textValue}
     </div>
   );
 };
 const Header = (props: IProps) => {
-  const { question, dataFormatType, setDataFormatType, isImg } = props;
+  const { question, dataFormatType, setDataFormatType, isImg, isAudio } = props;
   const DEFAULT_HEIGHT = 300;
   const { t } = useTranslation();
 
@@ -91,7 +107,12 @@ const Header = (props: IProps) => {
         />
       </div>
       <div className={`${LLMViewCls}__headerContent`}>
-        <RenderQuestion question={question} dataFormatType={dataFormatType} isImg={isImg} />
+        <RenderQuestion
+          question={question}
+          dataFormatType={dataFormatType}
+          isImg={isImg}
+          isAudio={isAudio}
+        />
       </div>
     </Resizable>
   );

diff --git a/packages/lb-components/src/components/LLMToolView/questionView/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/index.tsx
@@ -22,6 +22,7 @@ import DiffMatchPatchComponent from '@/components/diffMatchPatchComponent';
 import Header from './components/header';
 import ImgView from './components/imgView';
 import { isString } from 'lodash';
+import AudioView from './components/audioView';
 
 interface IProps {
   hoverKey?: number;
@@ -95,6 +96,8 @@ const QuestionView: React.FC<IProps> = (props) => {
   const [dataFormatType, setDataFormatType] = useState(EDataFormatType.Default);
   const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture;
   const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture;
+  const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio;
+  const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio;
   const { t } = useTranslation();
 
   useEffect(() => {
@@ -132,19 +135,31 @@ const QuestionView: React.FC<IProps> = (props) => {
       )}
     </div>
   );
+
+  const answerContent = () => {
+    if (answerIsAudio) {
+      return <AudioView hoverKey={hoverKey} answerList={answerList} />;
+    }
+    if (answerIsImg) {
+      return <ImgView hoverKey={hoverKey} answerList={answerList} />;
+    }
+    return textAnswer;
+  };
+
   return (
     <div className={LLMViewCls}>
       <Header
         question={question}
         dataFormatType={dataFormatType}
         setDataFormatType={setDataFormatType}
         isImg={questionIsImg}
+        isAudio={questionIsAudio}
       />
       <div className={`${LLMViewCls}__textBox`}>
         <div className={`${LLMViewCls}__title`}>
           {t('Answer')} {answerHeaderSlot}
         </div>
-        {answerIsImg ? <ImgView hoverKey={hoverKey} answerList={answerList} /> : textAnswer}
+        {answerContent()}
       </div>
     </div>
   );