From 8cfa154aa51144d79f591bcf89884d9afce9f130 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 07:20:26 -0500 Subject: [PATCH 01/23] add mistral to settings --- private_gpt/settings/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 8b03f6111..8fb85ed24 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -98,13 +98,14 @@ class LocalSettings(BaseModel): embedding_hf_model_name: str = Field( description="Name of the HuggingFace model to use for embeddings" ) - prompt_style: Literal["default", "llama2", "tag"] = Field( + prompt_style: Literal["default", "llama2", "tag", "mistral"] = Field( "llama2", description=( "The prompt style to use for the chat engine. " "If `default` - use the default prompt style from the llama_index. It should look like `role: message`.\n" "If `llama2` - use the llama2 prompt style from the llama_index. Based on ``, `[INST]` and `<>`.\n" "If `tag` - use the `tag` prompt style. It should look like `<|role|>: message`. \n" + "If `mistral` - use the `mistral prompt style. It shoudl look like [INST] {System Prompt} [/INST][INST] { UserInstructions } [/INST] "`llama2` is the historic behaviour. `default` might work better with your custom models." ), ) From 0ad6a52582a0f1e991a67fff2b07b8ee8453c6d0 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 07:32:07 -0500 Subject: [PATCH 02/23] add mistral template with system prompt --- private_gpt/components/llm/prompt_helper.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index a8ca60f27..2db7b39ea 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -2,6 +2,7 @@ import logging from collections.abc import Sequence from typing import Any, Literal +from private_gpt.settings.settings import settings from llama_index.llms import ChatMessage, MessageRole from llama_index.llms.llama_utils import ( @@ -122,9 +123,24 @@ def _completion_to_prompt(self, completion: str) -> str: [ChatMessage(content=completion, role=MessageRole.USER)] ) +class MistralPromptStyle(AbstractPromptStyle): + def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: + p = settings().ui.default_chat_system_prompt + prompt = f"{p.strip()}" + for message in messages: + role = message.role + content = message.content or "" + if role.lower() == "system": + message_from_user = f"[INST] {content.strip()} [/INST]" + prompt += message_from_user + elif role.lower() == "user": + prompt += "" + message_from_user = f"[INST] {content.strip()} [/INST]" + prompt += message_from_user + return prompt def get_prompt_style( - prompt_style: Literal["default", "llama2", "tag"] | None + prompt_style: Literal["default", "llama2", "tag", "mistral"] | None ) -> AbstractPromptStyle: """Get the prompt style to use from the given string. @@ -137,4 +153,6 @@ def get_prompt_style( return Llama2PromptStyle() elif prompt_style == "tag": return TagPromptStyle() + elif prompt_style == "mistral": + return MistralPromptStyle() raise ValueError(f"Unknown prompt_style='{prompt_style}'") From bef698cbe6ece466ebc47b63434b6a000f484b64 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 19:43:35 -0500 Subject: [PATCH 03/23] add mistral prompt template --- tests/test_prompt_helper.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 48cac0ba5..4c6885491 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -5,6 +5,7 @@ DefaultPromptStyle, Llama2PromptStyle, TagPromptStyle, + MistralPromptStyle, get_prompt_style, ) @@ -15,6 +16,7 @@ ("default", DefaultPromptStyle), ("llama2", Llama2PromptStyle), ("tag", TagPromptStyle), + ("mistral", MistralPromptStyle) ], ) def test_get_prompt_style_success(prompt_style, expected_prompt_style): @@ -61,6 +63,37 @@ def test_tag_prompt_style_format_with_system_prompt(): assert prompt_style.messages_to_prompt(messages) == expected_prompt +def test_mistral_prompt_style_format(): + prompt_style = MistralPromptStyle() + messages = [ + ChatMessage(content="You are an AI assistant.", role=MessageRole.SYSTEM), + ChatMessage(content="Hello, how are you doing?", role=MessageRole.USER), + ] + + expected_prompt = ( + "[INST] You are an AI assistant. [/INST]" + "[INST] Hello, how are you doing? [/INST]" + ) + + assert prompt_style.messages_to_prompt(messages) == expected_prompt + + +def test_mistral_prompt_style_format_with_system_prompt(): + prompt_style = MistralPromptStyle() + messages = [ + ChatMessage( + content="FOO BAR Custom sys prompt from messages.", role=MessageRole.SYSTEM + ), + ChatMessage(content="Hello, how are you doing?", role=MessageRole.USER), + ] + + expected_prompt = ( + "[INST] FOO BAR Custom sys prompt from messages. [/INST]" + "[INST] Hello, how are you doing? [/INST]" + ) + + assert prompt_style.messages_to_prompt(messages) == expected_prompt + def test_llama2_prompt_style_format(): prompt_style = Llama2PromptStyle() From b9a10b84d09eefda8b8e50d601a1bf36c810521c Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 19:57:10 -0500 Subject: [PATCH 04/23] correct tests for mistral test should always include system prompt.. --- tests/test_prompt_helper.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 4c6885491..851cbd8f5 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -1,5 +1,6 @@ import pytest from llama_index.llms import ChatMessage, MessageRole +from private_gpt.settings.settings import settings from private_gpt.components.llm.prompt_helper import ( DefaultPromptStyle, @@ -71,24 +72,7 @@ def test_mistral_prompt_style_format(): ] expected_prompt = ( - "[INST] You are an AI assistant. [/INST]" - "[INST] Hello, how are you doing? [/INST]" - ) - - assert prompt_style.messages_to_prompt(messages) == expected_prompt - - -def test_mistral_prompt_style_format_with_system_prompt(): - prompt_style = MistralPromptStyle() - messages = [ - ChatMessage( - content="FOO BAR Custom sys prompt from messages.", role=MessageRole.SYSTEM - ), - ChatMessage(content="Hello, how are you doing?", role=MessageRole.USER), - ] - - expected_prompt = ( - "[INST] FOO BAR Custom sys prompt from messages. [/INST]" + f"[INST] {p.strip()} [/INST][INST] You are an AI assistant. [/INST]" "[INST] Hello, how are you doing? [/INST]" ) From 9ceff4f83a3273dd4b44ad367cac5f89273f6467 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 21:19:11 -0500 Subject: [PATCH 05/23] add prompt from settings was in the other function I removed --- tests/test_prompt_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 851cbd8f5..ce27f8fe7 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -65,6 +65,7 @@ def test_tag_prompt_style_format_with_system_prompt(): assert prompt_style.messages_to_prompt(messages) == expected_prompt def test_mistral_prompt_style_format(): + p = settings().ui.default_chat_system_prompt prompt_style = MistralPromptStyle() messages = [ ChatMessage(content="You are an AI assistant.", role=MessageRole.SYSTEM), From e8d2e96697e0ee540d39039af84ff64bcd4f4b80 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 21:48:50 -0500 Subject: [PATCH 06/23] close string with " --- private_gpt/settings/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 8fb85ed24..b0f08eb2f 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -105,7 +105,7 @@ class LocalSettings(BaseModel): "If `default` - use the default prompt style from the llama_index. It should look like `role: message`.\n" "If `llama2` - use the llama2 prompt style from the llama_index. Based on ``, `[INST]` and `<>`.\n" "If `tag` - use the `tag` prompt style. It should look like `<|role|>: message`. \n" - "If `mistral` - use the `mistral prompt style. It shoudl look like [INST] {System Prompt} [/INST][INST] { UserInstructions } [/INST] + "If `mistral` - use the `mistral prompt style. It shoudl look like [INST] {System Prompt} [/INST][INST] { UserInstructions } [/INST]" "`llama2` is the historic behaviour. `default` might work better with your custom models." ), ) From 8d5c18b8fb09dcbc4f4293a88bf612c7ebde6f32 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 19 Dec 2023 23:04:36 -0500 Subject: [PATCH 07/23] add completion to prompt I am suggesting these changes from outside of my working environment, and it shows. --- private_gpt/components/llm/prompt_helper.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index 2db7b39ea..d7ebe4035 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -139,6 +139,11 @@ def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: prompt += message_from_user return prompt + def _completion_to_prompt(self, completion: str) -> str: + return self._messages_to_prompt( + [ChatMessage(content=completion, role=MessageRole.USER)] + ) + def get_prompt_style( prompt_style: Literal["default", "llama2", "tag", "mistral"] | None ) -> AbstractPromptStyle: From 29f616b04e5df1f241a238b82aeb3c857e20214a Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Wed, 20 Dec 2023 02:34:39 -0500 Subject: [PATCH 08/23] wrap system prompt [INST] --- private_gpt/components/llm/prompt_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index d7ebe4035..b3aa4df5e 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -126,7 +126,7 @@ def _completion_to_prompt(self, completion: str) -> str: class MistralPromptStyle(AbstractPromptStyle): def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: p = settings().ui.default_chat_system_prompt - prompt = f"{p.strip()}" + prompt = f"[INST] {p.strip()} [/INST]" for message in messages: role = message.role content = message.content or "" From 09bc556e375c28020a196a7d15224c956abe85d2 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Wed, 20 Dec 2023 02:45:04 -0500 Subject: [PATCH 09/23] sort imports --- tests/test_prompt_helper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index ce27f8fe7..b279b638d 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -1,15 +1,13 @@ import pytest from llama_index.llms import ChatMessage, MessageRole -from private_gpt.settings.settings import settings - from private_gpt.components.llm.prompt_helper import ( DefaultPromptStyle, Llama2PromptStyle, - TagPromptStyle, MistralPromptStyle, + TagPromptStyle, get_prompt_style, ) - +from private_gpt.settings.settings import settings @pytest.mark.parametrize( ("prompt_style", "expected_prompt_style"), From d19eee213259b69297879984e2b9ec1289a691a9 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Wed, 20 Dec 2023 02:49:52 -0500 Subject: [PATCH 10/23] space between external vs internal imports --- tests/test_prompt_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index b279b638d..96b714f90 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -1,5 +1,6 @@ import pytest from llama_index.llms import ChatMessage, MessageRole + from private_gpt.components.llm.prompt_helper import ( DefaultPromptStyle, Llama2PromptStyle, From 29611a7de5f1613b7687698b6ed4ac37fa9ed9ae Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Thu, 21 Dec 2023 00:15:50 -0500 Subject: [PATCH 11/23] ruff --fix --- private_gpt/components/llm/prompt_helper.py | 3 ++- scripts/extract_openapi.py | 1 + tests/test_prompt_helper.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index b3aa4df5e..ca18fac57 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -2,7 +2,6 @@ import logging from collections.abc import Sequence from typing import Any, Literal -from private_gpt.settings.settings import settings from llama_index.llms import ChatMessage, MessageRole from llama_index.llms.llama_utils import ( @@ -10,6 +9,8 @@ messages_to_prompt, ) +from private_gpt.settings.settings import settings + logger = logging.getLogger(__name__) diff --git a/scripts/extract_openapi.py b/scripts/extract_openapi.py index ba6f138ad..15840d91f 100644 --- a/scripts/extract_openapi.py +++ b/scripts/extract_openapi.py @@ -1,6 +1,7 @@ import argparse import json import sys + import yaml from uvicorn.importer import import_from_string diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 96b714f90..33bfc84c2 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -10,6 +10,7 @@ ) from private_gpt.settings.settings import settings + @pytest.mark.parametrize( ("prompt_style", "expected_prompt_style"), [ From 2ddc042396e8fa3870dd148fa09dea73a78cce8a Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Thu, 21 Dec 2023 00:21:50 -0500 Subject: [PATCH 12/23] poetry run black . --- private_gpt/components/llm/prompt_helper.py | 2 ++ tests/test_prompt_helper.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index ca18fac57..89dc49f57 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -124,6 +124,7 @@ def _completion_to_prompt(self, completion: str) -> str: [ChatMessage(content=completion, role=MessageRole.USER)] ) + class MistralPromptStyle(AbstractPromptStyle): def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: p = settings().ui.default_chat_system_prompt @@ -145,6 +146,7 @@ def _completion_to_prompt(self, completion: str) -> str: [ChatMessage(content=completion, role=MessageRole.USER)] ) + def get_prompt_style( prompt_style: Literal["default", "llama2", "tag", "mistral"] | None ) -> AbstractPromptStyle: diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 33bfc84c2..483a8d5d1 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -17,7 +17,7 @@ ("default", DefaultPromptStyle), ("llama2", Llama2PromptStyle), ("tag", TagPromptStyle), - ("mistral", MistralPromptStyle) + ("mistral", MistralPromptStyle), ], ) def test_get_prompt_style_success(prompt_style, expected_prompt_style): @@ -64,6 +64,7 @@ def test_tag_prompt_style_format_with_system_prompt(): assert prompt_style.messages_to_prompt(messages) == expected_prompt + def test_mistral_prompt_style_format(): p = settings().ui.default_chat_system_prompt prompt_style = MistralPromptStyle() From fbed2e8e22c9153435157797dbc3ba55b7e55028 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Thu, 21 Dec 2023 00:24:25 -0500 Subject: [PATCH 13/23] make mistral preset style --- settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.yaml b/settings.yaml index c3929539a..dd12d111a 100644 --- a/settings.yaml +++ b/settings.yaml @@ -47,7 +47,7 @@ qdrant: path: local_data/private_gpt/qdrant local: - prompt_style: "llama2" + prompt_style: "mistral" llm_hf_repo_id: TheBloke/Mistral-7B-Instruct-v0.2-GGUF llm_hf_model_file: mistral-7b-instruct-v0.2.Q4_K_M.gguf embedding_hf_model_name: BAAI/bge-small-en-v1.5 From 6f206b9c95a770e7e68b0a1c282e3f4feff48bcd Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Thu, 21 Dec 2023 02:42:14 -0500 Subject: [PATCH 14/23] remove redundant systemprompt --- private_gpt/components/llm/prompt_helper.py | 4 +--- tests/test_prompt_helper.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index 89dc49f57..71055de42 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -9,8 +9,6 @@ messages_to_prompt, ) -from private_gpt.settings.settings import settings - logger = logging.getLogger(__name__) @@ -128,7 +126,7 @@ def _completion_to_prompt(self, completion: str) -> str: class MistralPromptStyle(AbstractPromptStyle): def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: p = settings().ui.default_chat_system_prompt - prompt = f"[INST] {p.strip()} [/INST]" + prompt = f"" for message in messages: role = message.role content = message.content or "" diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 483a8d5d1..080fb5bc1 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -8,7 +8,6 @@ TagPromptStyle, get_prompt_style, ) -from private_gpt.settings.settings import settings @pytest.mark.parametrize( @@ -66,7 +65,6 @@ def test_tag_prompt_style_format_with_system_prompt(): def test_mistral_prompt_style_format(): - p = settings().ui.default_chat_system_prompt prompt_style = MistralPromptStyle() messages = [ ChatMessage(content="You are an AI assistant.", role=MessageRole.SYSTEM), @@ -74,7 +72,7 @@ def test_mistral_prompt_style_format(): ] expected_prompt = ( - f"[INST] {p.strip()} [/INST][INST] You are an AI assistant. [/INST]" + f"[INST] You are an AI assistant. [/INST]" "[INST] Hello, how are you doing? [/INST]" ) From 2f83b3b2ae9db6c6d99ba53b4437a7aa3a421875 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Thu, 21 Dec 2023 02:44:44 -0500 Subject: [PATCH 15/23] trailing artifacts (ruff) --- private_gpt/components/llm/prompt_helper.py | 3 +-- tests/test_prompt_helper.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index 71055de42..7461a7189 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -125,8 +125,7 @@ def _completion_to_prompt(self, completion: str) -> str: class MistralPromptStyle(AbstractPromptStyle): def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: - p = settings().ui.default_chat_system_prompt - prompt = f"" + prompt = "" for message in messages: role = message.role content = message.content or "" diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 080fb5bc1..cbd1f3bab 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -72,7 +72,7 @@ def test_mistral_prompt_style_format(): ] expected_prompt = ( - f"[INST] You are an AI assistant. [/INST]" + "[INST] You are an AI assistant. [/INST]" "[INST] Hello, how are you doing? [/INST]" ) From e168df6af620e7a1621758b1595ebb52c5066509 Mon Sep 17 00:00:00 2001 From: CognitiveTech Date: Tue, 16 Jan 2024 15:15:41 -0500 Subject: [PATCH 16/23] add chatml --- private_gpt/components/llm/prompt_helper.py | 24 ++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index 7461a7189..f8ae47c69 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -143,9 +143,29 @@ def _completion_to_prompt(self, completion: str) -> str: [ChatMessage(content=completion, role=MessageRole.USER)] ) +class ChatMLPromptStyle(AbstractPromptStyle): + def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: + prompt = "<|im_start|>system\n" + for message in messages: + role = message.role + content = message.content or "" + if role.lower() == "system": + message_from_user = f"{content.strip()}" + prompt += message_from_user + elif role.lower() == "user": + prompt += "<|im_end|>\n<|im_start|>user\n" + message_from_user = f"{content.strip()}<|im_end|>\n" + prompt += message_from_user + prompt += "<|im_start|>assistant\n" + return prompt + + def _completion_to_prompt(self, completion: str) -> str: + return self._messages_to_prompt( + [ChatMessage(content=completion, role=MessageRole.USER)] + ) def get_prompt_style( - prompt_style: Literal["default", "llama2", "tag", "mistral"] | None + prompt_style: Literal["default", "llama2", "tag", "mistral", "chatml"] | None ) -> AbstractPromptStyle: """Get the prompt style to use from the given string. @@ -160,4 +180,6 @@ def get_prompt_style( return TagPromptStyle() elif prompt_style == "mistral": return MistralPromptStyle() + elif prompt_style == "chatml": + return ChatMLPromptStyle() raise ValueError(f"Unknown prompt_style='{prompt_style}'") From 0ee082ff6e2f74b24a75fa4e9f25a67728d8ed56 Mon Sep 17 00:00:00 2001 From: CognitiveTech Date: Tue, 16 Jan 2024 15:16:32 -0500 Subject: [PATCH 17/23] chatml to settings.py --- private_gpt/settings/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index b0f08eb2f..f0470ad57 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -98,7 +98,7 @@ class LocalSettings(BaseModel): embedding_hf_model_name: str = Field( description="Name of the HuggingFace model to use for embeddings" ) - prompt_style: Literal["default", "llama2", "tag", "mistral"] = Field( + prompt_style: Literal["default", "llama2", "tag", "mistral", "chatml"] = Field( "llama2", description=( "The prompt style to use for the chat engine. " From af1590d502ecaef48cabdf91885f0f5ccadd2f38 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 16 Jan 2024 15:25:46 -0500 Subject: [PATCH 18/23] test chatml --- tests/test_prompt_helper.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index cbd1f3bab..35225e442 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -79,6 +79,24 @@ def test_mistral_prompt_style_format(): assert prompt_style.messages_to_prompt(messages) == expected_prompt +def test_chatml_prompt_style_format(): + prompt_style = MistralPromptStyle() + messages = [ + ChatMessage(content="You are an AI assistant.", role=MessageRole.SYSTEM), + ChatMessage(content="Hello, how are you doing?", role=MessageRole.USER), + ] + + expected_prompt = ( + "<|im_start|>system" + "You are an AI assistant.<|im_end|>" + "<|im_start|>user" + "Hello, how are you doing?<|im_end|>" + "<|im_start|>assistant\n" + ) + + assert prompt_style.messages_to_prompt(messages) == expected_prompt + + def test_llama2_prompt_style_format(): prompt_style = Llama2PromptStyle() messages = [ From 9a4822dc1a16b633b5f7bc2d84ae3b130f53a77e Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 16 Jan 2024 15:29:11 -0500 Subject: [PATCH 19/23] run black --- private_gpt/components/llm/prompt_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/private_gpt/components/llm/prompt_helper.py b/private_gpt/components/llm/prompt_helper.py index f8ae47c69..d1df9b814 100644 --- a/private_gpt/components/llm/prompt_helper.py +++ b/private_gpt/components/llm/prompt_helper.py @@ -143,6 +143,7 @@ def _completion_to_prompt(self, completion: str) -> str: [ChatMessage(content=completion, role=MessageRole.USER)] ) + class ChatMLPromptStyle(AbstractPromptStyle): def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: prompt = "<|im_start|>system\n" @@ -155,7 +156,7 @@ def _messages_to_prompt(self, messages: Sequence[ChatMessage]) -> str: elif role.lower() == "user": prompt += "<|im_end|>\n<|im_start|>user\n" message_from_user = f"{content.strip()}<|im_end|>\n" - prompt += message_from_user + prompt += message_from_user prompt += "<|im_start|>assistant\n" return prompt @@ -164,6 +165,7 @@ def _completion_to_prompt(self, completion: str) -> str: [ChatMessage(content=completion, role=MessageRole.USER)] ) + def get_prompt_style( prompt_style: Literal["default", "llama2", "tag", "mistral", "chatml"] | None ) -> AbstractPromptStyle: From 8c062c9b013a6f2cde5f63784a60aaac105a94cd Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 16 Jan 2024 15:40:58 -0500 Subject: [PATCH 20/23] passing --- tests/test_prompt_helper.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 35225e442..1f4c62ffd 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -6,6 +6,7 @@ Llama2PromptStyle, MistralPromptStyle, TagPromptStyle, + ChatMLPromptStyle, get_prompt_style, ) @@ -80,17 +81,17 @@ def test_mistral_prompt_style_format(): def test_chatml_prompt_style_format(): - prompt_style = MistralPromptStyle() + prompt_style = ChatMLPromptStyle() messages = [ ChatMessage(content="You are an AI assistant.", role=MessageRole.SYSTEM), ChatMessage(content="Hello, how are you doing?", role=MessageRole.USER), ] expected_prompt = ( - "<|im_start|>system" - "You are an AI assistant.<|im_end|>" - "<|im_start|>user" - "Hello, how are you doing?<|im_end|>" + "<|im_start|>system\n" + "You are an AI assistant.<|im_end|>\n" + "<|im_start|>user\n" + "Hello, how are you doing?<|im_end|>\n" "<|im_start|>assistant\n" ) From 3c363e0bf0d8da1fe65218c59ebc51bf07e8bfa9 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 16 Jan 2024 15:47:52 -0500 Subject: [PATCH 21/23] add to docs --- fern/docs/pages/recipes/list-llm.mdx | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/fern/docs/pages/recipes/list-llm.mdx b/fern/docs/pages/recipes/list-llm.mdx index 2cb80e483..1e53804bb 100644 --- a/fern/docs/pages/recipes/list-llm.mdx +++ b/fern/docs/pages/recipes/list-llm.mdx @@ -24,7 +24,7 @@ user: {{ user_message }} assistant: {{ assistant_message }} ``` -And the "`tag`" style looks like this: +The "`tag`" style looks like this: ```text <|system|>: {{ system_prompt }} @@ -32,7 +32,23 @@ And the "`tag`" style looks like this: <|assistant|>: {{ assistant_message }} ``` -Some LLMs will not understand this prompt style, and will not work (returning nothing). +The "`mistral`" style looks like this: + +```text +[INST] You are an AI assistant. [/INST][INST] Hello, how are you doing? [/INST] +``` + +The "`chatml`" style looks like this: +```text +<|im_start|>system +{{ system_prompt }}<|im_end|> +<|im_start|>user" +{{ user_message }}<|im_end|> +<|im_start|>assistant +{{ assistant_message }} +``` + +Some LLMs will not understand these prompt styles, and will not work (returning nothing). You can try to change the prompt style to `default` (or `tag`) in the settings, and it will change the way the messages are formatted to be passed to the LLM. From 6f21cfac3f45f61e26f99216321f8ffc471ef645 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 16 Jan 2024 15:57:22 -0500 Subject: [PATCH 22/23] ruff --- tests/test_prompt_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index 1f4c62ffd..b4a1a7e2a 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -2,11 +2,11 @@ from llama_index.llms import ChatMessage, MessageRole from private_gpt.components.llm.prompt_helper import ( + ChatMLPromptStyle, DefaultPromptStyle, Llama2PromptStyle, MistralPromptStyle, TagPromptStyle, - ChatMLPromptStyle, get_prompt_style, ) @@ -18,6 +18,7 @@ ("llama2", Llama2PromptStyle), ("tag", TagPromptStyle), ("mistral", MistralPromptStyle), + ("chatml", ChatMLPromptStyle) ], ) def test_get_prompt_style_success(prompt_style, expected_prompt_style): From 18380b57656f22a31d662d9ce2433002fd1314c2 Mon Sep 17 00:00:00 2001 From: cognitivetech Date: Tue, 16 Jan 2024 16:00:12 -0500 Subject: [PATCH 23/23] comma close --- tests/test_prompt_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_prompt_helper.py b/tests/test_prompt_helper.py index b4a1a7e2a..48597698b 100644 --- a/tests/test_prompt_helper.py +++ b/tests/test_prompt_helper.py @@ -18,7 +18,7 @@ ("llama2", Llama2PromptStyle), ("tag", TagPromptStyle), ("mistral", MistralPromptStyle), - ("chatml", ChatMLPromptStyle) + ("chatml", ChatMLPromptStyle), ], ) def test_get_prompt_style_success(prompt_style, expected_prompt_style):