diff --git a/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx b/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx
index b5c061da3..eb56737af 100644
--- a/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx
+++ b/packages/lb-components/src/components/LLMMultiWheelView/dialogView/index.tsx
@@ -1,4 +1,5 @@
import { getTextControlByConfig, RenderAnswer } from '@/components/LLMToolView/questionView';
+import AudioView from '@/components/LLMToolView/questionView/components/audioView';
import { RenderQuestion } from '@/components/LLMToolView/questionView/components/header';
import ImgView from '@/components/LLMToolView/questionView/components/imgView';
import { ILLMMultiWheelToolConfig } from '@/components/LLMToolView/types';
@@ -18,6 +19,8 @@ interface IDialogViewProps {
answerIsImg: boolean;
questionIsImg: boolean;
LLMConfig?: ILLMMultiWheelToolConfig;
+ answerIsAudio?: boolean;
+ questionIsAudio?: boolean;
}
const DialogView = (props: IDialogViewProps) => {
@@ -29,6 +32,8 @@ const DialogView = (props: IDialogViewProps) => {
name = '',
answerIsImg,
questionIsImg,
+ questionIsAudio,
+ answerIsAudio,
LLMConfig,
} = props;
const { t } = useTranslation();
@@ -37,6 +42,39 @@ const DialogView = (props: IDialogViewProps) => {
const order = index + 1;
const showName = name || `${t('Dialog')}${order}`;
+ const dialogAnswer = () => {
+ if (answerIsImg) {
+ return (
+ <>
+
+
+ >
+ );
+ }
+ if (answerIsAudio) {
+ return (
+ <>
+
+
+ >
+ );
+ }
+ return answerList?.map((item: any, index: number) => {
+ const order = index + 1;
+ const answer = { ...item, order };
+ const isTextControl = getTextControlByConfig(answer, LLMConfig);
+ return (
+
+
+
+
+ );
+ });
+ };
+
return (
{
question={question}
dataFormatType={dataFormatType}
isImg={questionIsImg}
+ isAudio={questionIsAudio}
/>
-
- {answerIsImg ? (
- <>
-
-
- >
- ) : (
- answerList?.map((item: any, index: number) => {
- const order = index + 1;
- const answer = { ...item, order };
- const isTextControl = getTextControlByConfig(answer, LLMConfig);
- return (
-
-
-
-
- );
- })
- )}
-
+ {dialogAnswer()}
);
};
diff --git a/packages/lb-components/src/components/LLMMultiWheelView/index.tsx b/packages/lb-components/src/components/LLMMultiWheelView/index.tsx
index 533db51e4..9fd84de48 100644
--- a/packages/lb-components/src/components/LLMMultiWheelView/index.tsx
+++ b/packages/lb-components/src/components/LLMMultiWheelView/index.tsx
@@ -30,10 +30,20 @@ interface ILLMMultiWheelSourceViewProps {
LLMConfig?: ILLMMultiWheelToolConfig;
dialogList: any[];
lang?: string;
+ questionIsAudio?: boolean;
+ answerIsAudio?: boolean;
}
export const LLMMultiWheelSourceView: React.FC = (props) => {
- const { questionIsImg, answerIsImg, LLMConfig, dialogList, lang = 'cn' } = props;
+ const {
+ questionIsImg,
+ answerIsImg,
+ questionIsAudio,
+ answerIsAudio,
+ LLMConfig,
+ dialogList,
+ lang = 'cn',
+ } = props;
const { dataFormatType, setDataFormatType } = useLLMMultiWheelStore();
useEffect(() => {
@@ -66,6 +76,8 @@ export const LLMMultiWheelSourceView: React.FC =
index={index}
questionIsImg={questionIsImg}
answerIsImg={answerIsImg}
+ questionIsAudio={questionIsAudio}
+ answerIsAudio={answerIsAudio}
LLMConfig={LLMConfig}
/>
))}
@@ -109,6 +121,9 @@ const LLMMultiWheelView: React.FC = (props) => {
const [dialogList, setDialogList] = useState([]);
const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture;
const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture;
+ const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio;
+ const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio;
+
const { t } = useTranslation();
const [, forceRender] = useState(0);
@@ -136,23 +151,25 @@ const LLMMultiWheelView: React.FC = (props) => {
const newDialogList = textList?.map((item: any, questionIndex: number) => {
return {
...item,
- question: questionIsImg
- ? getInfoFromLLMFile({
- type: 'question',
- questionIndex,
- llmFile,
- }) || item?.question
- : item?.question,
- answerList: item?.answerList?.map((i: any, answerIndex: number) => {
- const mapId = `${item?.id ?? ''}-${i?.id ?? ''}`;
- const info = answerIsImg
+ question:
+ questionIsImg || questionIsAudio
? getInfoFromLLMFile({
- type: 'answer',
+ type: 'question',
questionIndex,
- answerIndex,
llmFile,
- }) || {}
- : {};
+ }) || item?.question
+ : item?.question,
+ answerList: item?.answerList?.map((i: any, answerIndex: number) => {
+ const mapId = `${item?.id ?? ''}-${i?.id ?? ''}`;
+ const info =
+ answerIsImg || answerIsAudio
+ ? getInfoFromLLMFile({
+ type: 'answer',
+ questionIndex,
+ answerIndex,
+ llmFile,
+ }) || {}
+ : {};
return {
...i,
answer: i.answer,
@@ -164,7 +181,7 @@ const LLMMultiWheelView: React.FC = (props) => {
};
});
setDialogList(newDialogList);
- }, [newAnswerListMap, questionIsImg, answerIsImg]);
+ }, [newAnswerListMap, questionIsImg, questionIsAudio, answerIsImg]);
useEffect(() => {
if (stepList && step) {
@@ -192,6 +209,8 @@ const LLMMultiWheelView: React.FC = (props) => {
diff --git a/packages/lb-components/src/components/LLMToolView/index.tsx b/packages/lb-components/src/components/LLMToolView/index.tsx
index 27d358f96..477bed507 100644
--- a/packages/lb-components/src/components/LLMToolView/index.tsx
+++ b/packages/lb-components/src/components/LLMToolView/index.tsx
@@ -71,15 +71,17 @@ const LLMToolView: React.FC = (props) => {
return;
}
const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture;
+ const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio;
const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture;
+ const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio;
const qaData = imgList[imgIndex]?.questionList;
const llmFile = imgList[imgIndex]?.llmFile;
- const titleQuestion = questionIsImg ? llmFile?.question : qaData?.question;
+ const titleQuestion = questionIsImg || questionIsAudio ? llmFile?.question : qaData?.question;
setQuestion(titleQuestion);
let list = qaData?.answerList || [];
- if (answerIsImg) {
+ if (answerIsImg || answerIsAudio) {
list = llmFile?.answerList || [];
}
if (LLMConfig?.dataType?.response === ELLMDataType.None) {
diff --git a/packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx
new file mode 100644
index 000000000..1c6226c72
--- /dev/null
+++ b/packages/lb-components/src/components/LLMToolView/questionView/components/audioView/index.tsx
@@ -0,0 +1,51 @@
+/*
+ * @file LLM tool audio view
+ * @Author: lixinghua lixinghua@sensetime.com
+ * @Date: 2023-11-13
+ */
+import React from 'react';
+import { Tag } from 'antd';
+import { prefix } from '@/constant';
+import classNames from 'classnames';
+import { IAnswerList } from '@/components/LLMToolView/types';
+
+interface IProps {
+ hoverKey?: number;
+ answerList: IAnswerList[];
+}
+
+const LLMViewCls = `${prefix}-LLMView`;
+
+const AudioPlayer = React.memo(({ url }: { url?: string }) => (
+
+));
+
+const AudioView = (props: IProps) => {
+ const { answerList, hoverKey } = props;
+
+ return (
+
+ {answerList?.length > 0 &&
+ answerList.map((i: IAnswerList, index: number) => {
+ return (
+
+ );
+ })}
+
+ );
+};
+
+export default AudioView;
diff --git a/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx
index 0e21b7e44..7f1987f59 100644
--- a/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx
+++ b/packages/lb-components/src/components/LLMToolView/questionView/components/header/index.tsx
@@ -6,7 +6,7 @@
import React from 'react';
import { useTranslation } from 'react-i18next';
import { Resizable } from 're-resizable';
-import { Radio, Image } from 'antd';
+import { Radio, Image, Empty } from 'antd';
import { EDataFormatType, prefix } from '@/constant';
import { FileTextOutlined } from '@ant-design/icons';
import MarkdownView from '@/components/markdownView';
@@ -28,6 +28,7 @@ interface IProps {
dataFormatType: EDataFormatType;
setDataFormatType: (v: EDataFormatType) => void;
isImg?: boolean;
+ isAudio?: boolean;
}
const LLMViewCls = `${prefix}-LLMView`;
@@ -36,6 +37,7 @@ export const RenderQuestion = ({
question,
dataFormatType,
isImg,
+ isAudio,
}: {
question:
| string
@@ -48,6 +50,7 @@ export const RenderQuestion = ({
};
dataFormatType: EDataFormatType;
isImg?: boolean;
+ isAudio?: boolean;
}) => {
const textValue = isString(question) ? question : '';
const ImgFail = i18n.language === 'en' ? ImgFailEn : ImgFailCn;
@@ -56,6 +59,19 @@ export const RenderQuestion = ({
const url = isObject(question) ? question?.url : '';
return ;
}
+
+ if (isAudio) {
+ const url = isObject(question) ? question?.url : '';
+ if (url) {
+ return (
+
+ );
+ }
+ return ;
+ }
+
return (
{dataFormatType === EDataFormatType.Markdown ? : textValue}
@@ -63,7 +79,7 @@ export const RenderQuestion = ({
);
};
const Header = (props: IProps) => {
- const { question, dataFormatType, setDataFormatType, isImg } = props;
+ const { question, dataFormatType, setDataFormatType, isImg, isAudio } = props;
const DEFAULT_HEIGHT = 300;
const { t } = useTranslation();
@@ -91,7 +107,12 @@ const Header = (props: IProps) => {
/>
-
+
);
diff --git a/packages/lb-components/src/components/LLMToolView/questionView/index.tsx b/packages/lb-components/src/components/LLMToolView/questionView/index.tsx
index db0f0eb3c..a3b08030a 100644
--- a/packages/lb-components/src/components/LLMToolView/questionView/index.tsx
+++ b/packages/lb-components/src/components/LLMToolView/questionView/index.tsx
@@ -22,6 +22,7 @@ import DiffMatchPatchComponent from '@/components/diffMatchPatchComponent';
import Header from './components/header';
import ImgView from './components/imgView';
import { isString } from 'lodash';
+import AudioView from './components/audioView';
interface IProps {
hoverKey?: number;
@@ -95,6 +96,8 @@ const QuestionView: React.FC = (props) => {
const [dataFormatType, setDataFormatType] = useState(EDataFormatType.Default);
const questionIsImg = LLMConfig?.dataType?.prompt === ELLMDataType.Picture;
const answerIsImg = LLMConfig?.dataType?.response === ELLMDataType.Picture;
+ const questionIsAudio = LLMConfig?.dataType?.prompt === ELLMDataType.Audio;
+ const answerIsAudio = LLMConfig?.dataType?.response === ELLMDataType.Audio;
const { t } = useTranslation();
useEffect(() => {
@@ -132,6 +135,17 @@ const QuestionView: React.FC = (props) => {
)}
);
+
+ const answerContent = () => {
+ if (answerIsAudio) {
+ return ;
+ }
+ if (answerIsImg) {
+ return ;
+ }
+ return textAnswer;
+ };
+
return (
);
diff --git a/packages/lb-components/src/constant/index.tsx b/packages/lb-components/src/constant/index.tsx
index 932cf2968..2cad0e717 100644
--- a/packages/lb-components/src/constant/index.tsx
+++ b/packages/lb-components/src/constant/index.tsx
@@ -34,4 +34,5 @@ export enum ELLMDataType {
Picture = 'picture',
Text = 'text',
None = 'none',
+ Audio = 'audio'
}