Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

llama-cpp not working with current setup/deployment. #1

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

llama-cpp not working with current setup/deployment. #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
05a553a
Update launch.sh
OG-Sadpanda Jan 17, 2025
3d37681
Update requirements.txt
OG-Sadpanda Jan 17, 2025
d373323
Update launch.sh
OG-Sadpanda Jan 17, 2025
b384c31
adding deepseek-r1 and updating openchat
Jan 23, 2025
3f706d3
fixing deepseek settings
Jan 24, 2025
ab82825
fixing deepseek settings
Jan 24, 2025
28febcb
fixing settings
Jan 27, 2025
a521e93
fixing deepseek model
Jan 27, 2025
7cfc83d
removing deepseek
Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions launch.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@
cd ragnarok
python3 -m venv venv
source venv/bin/activate
# default llama-cpp-python does not support cuda, have to provide cmake_args.
export LLAMA_CUBLAS=1
CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip3 install llama-cpp-python==0.2.26

# You can change the visible device values here to specify what GPU(s) that you want RAGnarok to use (or not use).
# For this example I'm only allowing RAGnarok to use GPU device 0.
# See https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars for more details.
export CUDA_VISIBLE_DEVICES=0

pip3 install -r requirements.txt

# kick off the main app
Expand Down
4 changes: 2 additions & 2 deletions ragnarok/RAGnarok_Settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,14 @@ def wait_for_nemesis(nemesis_url, nemesis_user, nemesis_password, wait_timeout =
default_index = 0
if "llm_model" in cookies:
if "neural-chat" in cookies["llm_model"].lower():
default_index = 0
default_index = 0
elif "openchat" in cookies["llm_model"].lower():
default_index = 1
elif "starling" in cookies["llm_model"].lower():
default_index = 2
llm_model = st.selectbox(
label='LLM model to use',
options=('Intel/neural-chat-7b-v3-3', 'openchat-3.5-0106', 'Starling-LM-7B-alpha'),
options=('Intel/neural-chat-7b-v3-3', 'openchat-3.5-1210', 'Starling-LM-7B-alpha'),
help="The core LLM to use for chat over retrieved document snippets.",
index=default_index
)
Expand Down
8 changes: 4 additions & 4 deletions ragnarok/pages/1_RAGnarok_Chat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ def get_reranker(reranking_model, device):
try:
if cookies["llm_model"] == "Intel/neural-chat-7b-v3-3":
llm_model_path = hf_hub_download("TheBloke/neural-chat-7B-v3-3-GGUF", filename="neural-chat-7b-v3-3.Q5_K_M.gguf", local_files_only=True)
elif cookies["llm_model"] == "openchat-3.5-0106":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-0106-GGUF", filename="openchat-3.5-0106.Q5_K_M.gguf", local_files_only=True)
elif cookies["llm_model"] == "openchat-3.5-1210":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-1210-GGUF", filename="openchat-3.5-1210.Q8_0.gguf", local_files_only=True)
elif cookies["llm_model"] == "Starling-LM-7B-alpha":
llm_model_path = hf_hub_download("TheBloke/Starling-LM-7B-alpha-GGUF", filename="starling-lm-7b-alpha.Q5_K_M.gguf", local_files_only=True)
else:
Expand All @@ -113,8 +113,8 @@ def get_reranker(reranking_model, device):
with st.spinner("Downloading LLM model (this will take some time)..."):
if cookies["llm_model"] == "Intel/neural-chat-7b-v3-3":
llm_model_path = hf_hub_download("TheBloke/neural-chat-7B-v3-3-GGUF", filename="neural-chat-7b-v3-3.Q5_K_M.gguf")
elif cookies["llm_model"] == "openchat-3.5-0106":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-0106-GGUF", filename="openchat-3.5-0106.Q5_K_M.gguf")
elif cookies["llm_model"] == "openchat-3.5-1210":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-1210-GGUF", filename="openchat-3.5-1210.Q8_0.gguf")
elif cookies["llm_model"] == "Starling-LM-7B-alpha":
llm_model_path = hf_hub_download("TheBloke/Starling-LM-7B-alpha-GGUF", filename="starling-lm-7b-alpha.Q5_K_M.gguf")
else:
Expand Down
8 changes: 4 additions & 4 deletions ragnarok/pages/2_Free_Chat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ def get_llm(llm_model_path, n_gpu_layers):
try:
if cookies["llm_model"] == "Intel/neural-chat-7b-v3-3":
llm_model_path = hf_hub_download("TheBloke/neural-chat-7B-v3-3-GGUF", filename="neural-chat-7b-v3-3.Q5_K_M.gguf", local_files_only=True)
elif cookies["llm_model"] == "openchat-3.5-0106":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-0106-GGUF", filename="openchat-3.5-0106.Q5_K_M.gguf", local_files_only=True)
elif cookies["llm_model"] == "openchat-3.5-1210":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-1210-GGUF", filename="openchat-3.5-1210.Q8_0.gguf", local_files_only=True)
elif cookies["llm_model"] == "Starling-LM-7B-alpha":
llm_model_path = hf_hub_download("TheBloke/Starling-LM-7B-alpha-GGUF", filename="starling-lm-7b-alpha.Q5_K_M.gguf", local_files_only=True)
else:
Expand All @@ -63,8 +63,8 @@ def get_llm(llm_model_path, n_gpu_layers):
with st.spinner("Downloading LLM model (this will take some time)..."):
if cookies["llm_model"] == "Intel/neural-chat-7b-v3-3":
llm_model_path = hf_hub_download("TheBloke/neural-chat-7B-v3-3-GGUF", filename="neural-chat-7b-v3-3.Q5_K_M.gguf")
elif cookies["llm_model"] == "openchat-3.5-0106":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-0106-GGUF", filename="openchat-3.5-0106.Q5_K_M.gguf")
elif cookies["llm_model"] == "openchat-3.5-1210":
llm_model_path = hf_hub_download("TheBloke/openchat-3.5-1210-GGUF", filename="openchat-3.5-1210.Q8_0.gguf")
elif cookies["llm_model"] == "Starling-LM-7B-alpha":
llm_model_path = hf_hub_download("TheBloke/Starling-LM-7B-alpha-GGUF", filename="starling-lm-7b-alpha.Q5_K_M.gguf")
else:
Expand Down
3 changes: 1 addition & 2 deletions ragnarok/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ sentence-transformers==2.2.2
huggingface-hub==0.20.3
langchain==0.1.2
langchain-community==0.0.14
llama-cpp-python==0.2.26
streamlit==1.30.0
streamlit-extras==0.3.6
streamlit-cookies-manager==0.2.0
streamlit-cookies-manager==0.2.0