Release v2.4.1

.github/doxygen/Doxyfile

@@ -48,7 +48,7 @@ PROJECT_NAME           = "LLM for Unity"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER = v2.4.0
+PROJECT_NUMBER = v2.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

CHANGELOG.md

@@ -1,3 +1,13 @@
+## v2.4.1
+#### 🚀 Features
+
+- Static library linking on mobile (fixes iOS signing) (PR: #289)
+
+#### 🐛 Fixes
+
+- Fix support for extras (flash attention, iQ quants)  (PR: #292)
+
+
 ## v2.4.0
 #### 🚀 Features
 

CHANGELOG.release.md

@@ -1,16 +1,8 @@
 ### 🚀 Features
 
-- iOS deployment (PR: #267)
-- Improve building process (PR: #282)
-- Add structured output / function calling sample (PR: #281)
-- Update LlamaLib to v1.2.0 (llama.cpp b4218) (PR: #283)
+- Static library linking on mobile (fixes iOS signing) (PR: #289)
 
 ### 🐛 Fixes
 
-- Clear temp build directory before building (PR: #278)
-
-### 📦 General
-
-- Remove support for extras (flash attention, iQ quants) (PR: #284)
-- remove support for LLM base prompt (PR: #285)
+- Fix support for extras (flash attention, iQ quants)  (PR: #292)
 

Editor/LLMBuildProcessor.cs

@@ -2,6 +2,9 @@
 using UnityEditor.Build;
 using UnityEditor.Build.Reporting;
 using UnityEngine;
+#if UNITY_IOS
+using UnityEditor.iOS.Xcode;
+#endif
 
 namespace LLMUnity
 {
@@ -43,9 +46,27 @@ private void OnBuildError(string condition, string stacktrace, LogType type)
             if (type == LogType.Error) BuildCompleted();
         }
 
+#if UNITY_IOS
+        /// <summary>
+        /// Adds the Accelerate framework (for ios)
+        /// </summary>
+        public static void AddAccelerate(string outputPath)
+        {
+            string projPath = PBXProject.GetPBXProjectPath(outputPath);
+            PBXProject proj = new PBXProject();
+            proj.ReadFromFile(projPath);
+            proj.AddFrameworkToProject(proj.GetUnityMainTargetGuid(), "Accelerate.framework", false);
+            proj.AddFrameworkToProject(proj.GetUnityFrameworkTargetGuid(), "Accelerate.framework", false);
+            proj.WriteToFile(projPath);
+        }
+#endif
+
         // called after the build
         public void OnPostprocessBuild(BuildReport report)
         {
+#if UNITY_IOS
+            AddAccelerate(report.summary.outputPath);
+#endif
             BuildCompleted();
         }
 

Editor/LLMEditor.cs

@@ -111,6 +111,7 @@ public override void AddModelSettings(SerializedObject llmScriptSO)
             if (llmScriptSO.FindProperty("advancedOptions").boolValue)
             {
                 attributeClasses.Add(typeof(ModelAdvancedAttribute));
+                if (LLMUnitySetup.FullLlamaLib) attributeClasses.Add(typeof(ModelExtrasAttribute));
             }
             ShowPropertiesOfClass("", llmScriptSO, attributeClasses, false);
             Space();
@@ -444,12 +445,18 @@ private void CopyToClipboard(string text)
             te.Copy();
         }
 
+        public void AddExtrasToggle()
+        {
+            if (ToggleButton("Use extras", LLMUnitySetup.FullLlamaLib)) LLMUnitySetup.SetFullLlamaLib(!LLMUnitySetup.FullLlamaLib);
+        }
+
         public override void AddOptionsToggles(SerializedObject llmScriptSO)
         {
             AddDebugModeToggle();
 
             EditorGUILayout.BeginHorizontal();
             AddAdvancedOptionsToggle(llmScriptSO);
+            AddExtrasToggle();
             EditorGUILayout.EndHorizontal();
             Space();
         }

README.md

@@ -499,7 +499,8 @@ Save the scene, run and enjoy!
 ### LLM Settings
 
 - `Show/Hide Advanced Options` Toggle to show/hide advanced options from below
-- `Log Level` select how verbose the log messages arequants)
+- `Log Level` select how verbose the log messages are
+- `Use extras` select to install and allow the use of extra features (flash attention and IQ quants)
 
 #### 💻 Setup Settings
 
@@ -550,13 +551,15 @@ If the user's GPU is not supported, the LLM will fall back to the CPU
   - `Chat Template` the chat template being used for the LLM
   - `Lora` the path of the LoRAs being used (relative to the Assets/StreamingAssets folder)
   - `Lora Weights` the weights of the LoRAs being used
+  - `Flash Attention` click to use flash attention in the model (if `Use extras` is enabled)
 
 </details>
 
 ### LLMCharacter Settings
 
 - `Show/Hide Advanced Options` Toggle to show/hide advanced options from below
 - `Log Level` select how verbose the log messages are
+- `Use extras` select to install and allow the use of extra features (flash attention and IQ quants)
 
 #### 💻 Setup Settings
 <div>

Runtime/LLM.cs

@@ -59,6 +59,8 @@ public class LLM : MonoBehaviour
         [ModelAdvanced] public string lora = "";
         /// <summary> the weights of the LORA models being used.</summary>
         [ModelAdvanced] public string loraWeights = "";
+        /// <summary> enable use of flash attention </summary>
+        [ModelExtras] public bool flashAttention = false;
 
         /// <summary> API key to use for the server (optional) </summary>
         public string APIKey;
@@ -430,6 +432,7 @@ protected virtual string GetLlamaccpArguments()
             if (numThreadsToUse > 0) arguments += $" -t {numThreadsToUse}";
             arguments += loraArgument;
             arguments += $" -ngl {numGPULayers}";
+            if (LLMUnitySetup.FullLlamaLib && flashAttention) arguments += $" --flash-attn";
             if (remote)
             {
                 arguments += $" --port {port} --host 0.0.0.0";
@@ -719,7 +722,7 @@ public void ApplyLoras()
             json = json.Substring(startIndex, endIndex - startIndex);
 
             IntPtr stringWrapper = llmlib.StringWrapper_Construct();
-            llmlib.LLM_Lora_Weight(LLMObject, json, stringWrapper);
+            llmlib.LLM_LoraWeight(LLMObject, json, stringWrapper);
             llmlib.StringWrapper_Delete(stringWrapper);
         }
 

Runtime/LLMBuilder.cs

@@ -17,6 +17,7 @@ public class LLMBuilder
         static List<StringPair> movedPairs = new List<StringPair>();
         public static string BuildTempDir = Path.Combine(Application.temporaryCachePath, "LLMUnityBuild");
         public static string androidPluginDir = Path.Combine(Application.dataPath, "Plugins", "Android", "LLMUnity");
+        public static string iOSPluginDir = Path.Combine(Application.dataPath, "Plugins", "iOS", "LLMUnity");
         static string movedCache = Path.Combine(BuildTempDir, "moved.json");
 
         [InitializeOnLoadMethod]
@@ -87,7 +88,7 @@ public static void MovePath(string source, string target)
         /// <param name="path">path</param>
         public static bool DeletePath(string path)
         {
-            string[] allowedDirs = new string[] { LLMUnitySetup.GetAssetPath(), BuildTempDir, androidPluginDir};
+            string[] allowedDirs = new string[] { LLMUnitySetup.GetAssetPath(), BuildTempDir, androidPluginDir, iOSPluginDir};
             bool deleteOK = false;
             foreach (string allowedDir in allowedDirs) deleteOK = deleteOK || LLMUnitySetup.IsSubPath(path, allowedDir);
             if (!deleteOK)
@@ -148,10 +149,10 @@ static void AddActionAddMeta(string target)
         }
 
         /// <summary>
-        /// Hides all the library platforms apart from the target platform by moving out their library folders outside of StreamingAssets
+        /// Moves libraries in the correct place for building
         /// </summary>
         /// <param name="platform">target platform</param>
-        public static void HideLibraryPlatforms(string platform)
+        public static void BuildLibraryPlatforms(string platform)
         {
             List<string> platforms = new List<string>(){ "windows", "macos", "linux", "android", "ios", "setup" };
             platforms.Remove(platform);
@@ -161,6 +162,8 @@ public static void HideLibraryPlatforms(string platform)
                 foreach (string platformPrefix in platforms)
                 {
                     bool move = sourceName.StartsWith(platformPrefix);
+                    move = move || (sourceName.Contains("cuda") && !sourceName.Contains("full") && LLMUnitySetup.FullLlamaLib);
+                    move = move || (sourceName.Contains("cuda") && sourceName.Contains("full") && !LLMUnitySetup.FullLlamaLib);
                     if (move)
                     {
                         string target = Path.Combine(BuildTempDir, sourceName);
@@ -170,13 +173,14 @@ public static void HideLibraryPlatforms(string platform)
                 }
             }
 
-            if (platform == "android")
+            if (platform == "android" || platform == "ios")
             {
-                string source = Path.Combine(LLMUnitySetup.libraryPath, "android");
-                string target = Path.Combine(androidPluginDir, LLMUnitySetup.libraryName);
+                string pluginDir = platform == "android"? androidPluginDir: iOSPluginDir;
+                string source = Path.Combine(LLMUnitySetup.libraryPath, platform);
+                string target = Path.Combine(pluginDir, LLMUnitySetup.libraryName);
                 MoveAction(source, target);
                 MoveAction(source + ".meta", target + ".meta");
-                AddActionAddMeta(androidPluginDir);
+                AddActionAddMeta(pluginDir);
             }
         }
 
@@ -196,7 +200,7 @@ public static void Build(string platform)
         {
             DeletePath(BuildTempDir);
             Directory.CreateDirectory(BuildTempDir);
-            HideLibraryPlatforms(platform);
+            BuildLibraryPlatforms(platform);
             BuildModels();
         }
 

Runtime/LLMCaller.cs

@@ -128,7 +128,11 @@ protected virtual void AssignLLM()
             if (remote || llm != null) return;
 
             List<LLM> validLLMs = new List<LLM>();
+#if UNITY_6000_0_OR_NEWER
+            foreach (LLM foundllm in FindObjectsByType(typeof(LLM), FindObjectsSortMode.None))
+#else
             foreach (LLM foundllm in FindObjectsOfType<LLM>())
+#endif
             {
                 if (IsValidLLM(foundllm) && IsAutoAssignableLLM(foundllm)) validLLMs.Add(foundllm);
             }
@@ -284,7 +288,9 @@ protected virtual async Task<Ret> PostRequestRemote<Res, Ret>(string json, strin
                     }
 
                     // Start the request asynchronously
-                    var asyncOperation = request.SendWebRequest();
+                    UnityWebRequestAsyncOperation asyncOperation = request.SendWebRequest();
+                    await Task.Yield(); // Wait for the next frame so that asyncOperation is properly registered (especially if not in main thread)
+
                     float lastProgress = 0f;
                     // Continue updating progress until the request is completed
                     while (!asyncOperation.isDone)