Skip to content

Commit

Permalink
Automatically set context size for models when it is too large (>32768)
Browse files Browse the repository at this point in the history
  • Loading branch information
amakropoulos committed Sep 26, 2024
1 parent 8b3f7dc commit 40e68cd
Showing 1 changed file with 9 additions and 2 deletions.
11 changes: 9 additions & 2 deletions Runtime/LLM.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public class LLM : MonoBehaviour
[SerializeField]
private string SSLKey = "";
public string SSLKeyPath = "";
public static int maxContextLength = 32768;

/// \cond HIDE

Expand Down Expand Up @@ -211,9 +212,15 @@ public void SetModel(string path)
ModelEntry modelEntry = LLMManager.Get(model);
if (modelEntry == null) modelEntry = new ModelEntry(GetLLMManagerAssetRuntime(model));
SetTemplate(modelEntry.chatTemplate);
if (contextSize == 0 && modelEntry.contextLength > 32768)
if (contextSize > modelEntry.contextLength)
{
LLMUnitySetup.LogWarning($"The model {path} has very large context size ({modelEntry.contextLength}), consider setting it to a smaller value (<=32768) to avoid filling up the RAM");
contextSize = 0;
LLMUnitySetup.LogWarning($"The selected context size {contextSize} is larger than the context size supported from the {path} model ({modelEntry.contextLength}), resetting it to use the model's default");
}
if (contextSize == 0 && modelEntry.contextLength > maxContextLength)
{
contextSize = maxContextLength;
LLMUnitySetup.LogWarning($"The {path} model has very large context size ({modelEntry.contextLength}), it was automatically set to {maxContextLength} to avoid filling up the RAM");
}
}
#if UNITY_EDITOR
Expand Down

0 comments on commit 40e68cd

Please sign in to comment.