Llava Initial approach to clear images

LLama.Examples/Examples/LlavaInteractiveModeExecute.cs

@@ -1,8 +1,7 @@
 using System.Text.RegularExpressions;
-using LLama.Batched;
 using LLama.Common;
 using Spectre.Console;
-using LLama.Abstractions;
+using LLama.Native;
 
 namespace LLama.Examples.Examples
 {
@@ -19,12 +18,8 @@ public static async Task Run()
 
             var prompt = $"{{{modelImage}}}\nUSER:\nProvide a full description of the image.\nASSISTANT:\n";
 
-            var parameters = new ModelParams(modelPath)
-            {
-                ContextSize = 4096,
-                Seed = 1337,
-                GpuLayerCount = 10
-            };
+            var parameters = new ModelParams(modelPath);
+
             using var model = LLamaWeights.LoadFromFile(parameters);
             using var context = model.CreateContext(parameters);
 
@@ -47,16 +42,16 @@ public static async Task Run()
                 var imageMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
                 var imageCount = imageMatches.Count();
                 var hasImages = imageCount > 0;
-                byte[][] imageBytes = null;
 
                 if (hasImages)
                 {
                     var imagePathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
-                    var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value);
+                    var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value).ToList();
 
+                    List<byte[]> imageBytes;
                     try
                     {
-                        imageBytes = imagePaths.Select(File.ReadAllBytes).ToArray();
+                        imageBytes = imagePaths.Select(File.ReadAllBytes).ToList();
                     }
                     catch (IOException exception)
                     {
@@ -69,15 +64,17 @@ public static async Task Run()
                         break;
                     }
 
+                    // Each prompt with images we clear cache
+                    // When the prompt contains images we clear KV_CACHE to restart conversation
+                    // See:
+                    // https://github.com/ggerganov/llama.cpp/discussions/3620
+                    ex.Context.NativeHandle.KvCacheRemove( LLamaSeqId.Zero, -1, -1 );
 
                     int index = 0;
                     foreach (var path in imagePathsWithCurlyBraces)
                     {
                         // First image replace to tag <image, the rest of the images delete the tag
-                        if (index++ == 0)
-                            prompt = prompt.Replace(path, "<image>");
-                        else
-                            prompt = prompt.Replace(path, "");
+                        prompt = prompt.Replace(path, index++ == 0 ? "<image>" : "");
                     }
 
 
@@ -102,7 +99,7 @@ public static async Task Run()
                     //
                     foreach (var image in imagePaths)
                     {
-                        ex.Images.Add(File.ReadAllBytes(image));
+                        ex.Images.Add(await File.ReadAllBytesAsync(image));
                     }
                 }
 
@@ -118,7 +115,7 @@ public static async Task Run()
 
                 // let the user finish with exit
                 //
-                if (prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase))
+                if (prompt != null && prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase))
                     break;
 
             }

LLama/Abstractions/ILLamaExecutor.cs

@@ -25,7 +25,7 @@ public interface ILLamaExecutor
         public LLavaWeights? ClipModel { get;  }
 
         /// <summary>
-        /// List of images: Image filen path, uri or image byte array. See ImageData.
+        /// List of images: List of images in byte array format.
         /// </summary>
         public List<byte[]> Images { get; }
 

LLama/LLamaExecutorBase.cs

@@ -79,7 +79,7 @@
         public LLavaWeights? ClipModel { get;  }
 
         /// <inheritdoc />
-        public List<byte[]> Images { get; set; }
+        public List<byte[]> Images { get; }
 
         /// <summary>
         /// Current "mu" value for mirostat sampling
@@ -419,10 +419,10 @@
             public string? SessionFilePath { get; set; }
 
             [JsonPropertyName("embd")]
             public LLamaToken[] Embeds { get; set; }
 
             [JsonPropertyName("embd_inps")]
             public LLamaToken[] EmbedInps { get; set; }
 
             [JsonPropertyName("session_tokens")]
             public LLamaToken[] SessionTokens { get; set; }

LLama/LLamaInteractExecutor.cs

@@ -11,7 +11,7 @@
 using LLama.Exceptions;
 using LLama.Extensions;
 using Microsoft.Extensions.Logging;
-using System.Net.Http;
+
 
 namespace LLama
 {
@@ -101,7 +101,7 @@
             using (var fs = new FileStream(filename, FileMode.Open, FileAccess.Read))
             {
                 var state = await JsonSerializer.DeserializeAsync<InteractiveExecutorState>(fs);
                 await LoadState(state);
             }
         }
 
@@ -136,24 +136,33 @@
                     text += "\n";
                 }
 
-                var line_inp = Context.Tokenize(text, false);
-                _embed_inps.AddRange(line_inp);
-                args.RemainedTokens -= line_inp.Length;
+                if (!this.IsMultiModal)
+                {
+                    var line_inp = Context.Tokenize(text, false);
+                    _embed_inps.AddRange(line_inp);
+                    args.RemainedTokens -= line_inp.Length;
+                }
+                else
+                {
+                    PreprocessLlava(text, args, false);
+                }
             }
 
             return Task.CompletedTask;
         }
 
+        /// <inheritdoc />
         private Task PreprocessLlava(string text, InferStateArgs args, bool addBos = true )
         {
             int usedTokens = 0;
+
             // If the prompt contains the tag <image> extract this.
             _imageInPrompt = text.Contains("<image>");
-            if (_imageInPrompt && ClipModel != null)
+            if (_imageInPrompt && IsMultiModal )
             {
                 foreach (var image in Images)
                 {
                     _imageEmbedHandles.Add(SafeLlavaImageEmbedHandle.CreateFromMemory(ClipModel.NativeHandle, Context, image));
                 }
 
                 int imageIndex = text.IndexOf("<image>");
@@ -170,7 +179,16 @@
             }
             else
             {
-                _embed_inps = Context.Tokenize(text, true).ToList();
+                if (addBos)
+                {
+                    _embed_inps = Context.Tokenize(text, true).ToList();
+                }
+                else
+                {
+                    var line_inp = Context.Tokenize(text, false);
+                    _embed_inps.AddRange(line_inp);
+                    args.RemainedTokens -= line_inp.Length;                    
+                }
             }
             return Task.CompletedTask;
         }
@@ -181,11 +199,11 @@
         /// <param name="inferenceParams"></param>
         /// <param name="args"></param>
         /// <returns></returns>
         protected override async Task<(bool, IReadOnlyList<string>)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args)
         {
             if (_embed_inps.Count <= _consumedTokensCount)
             {
                 if (_last_n_tokens.TokensEndsWithAnyString(args.Antiprompts, Context.NativeHandle.ModelHandle, Context.Encoding))
                     args.WaitForInput = true;
 
                 if (_pastTokensCount > 0 && args.WaitForInput)
@@ -239,6 +257,7 @@
 
                     _EmbedImagePosition = -1;
                     _imageEmbedHandles.Clear();
+                    Images.Clear();
                 }
                 else
                 {

docs/Examples/LLavaInteractiveModeExecute.md

@@ -2,9 +2,9 @@
 
 ```cs
 using System.Text.RegularExpressions;
-using LLama.Batched;
 using LLama.Common;
 using Spectre.Console;
+using LLama.Native;
 
 namespace LLama.Examples.Examples
 {
@@ -21,11 +21,8 @@ namespace LLama.Examples.Examples
 
             var prompt = $"{{{modelImage}}}\nUSER:\nProvide a full description of the image.\nASSISTANT:\n";
 
-            var parameters = new ModelParams(modelPath)
-            {
-                ContextSize = 4096,
-                Seed = 1337,
-            };
+            var parameters = new ModelParams(modelPath);
+
             using var model = LLamaWeights.LoadFromFile(parameters);
             using var context = model.CreateContext(parameters);
 
@@ -48,16 +45,16 @@ namespace LLama.Examples.Examples
                 var imageMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
                 var imageCount = imageMatches.Count();
                 var hasImages = imageCount > 0;
-                byte[][] imageBytes = null;
 
                 if (hasImages)
                 {
                     var imagePathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
-                    var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value);
+                    var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value).ToList();
 
+                    List<byte[]> imageBytes;
                     try
                     {
-                        imageBytes = imagePaths.Select(File.ReadAllBytes).ToArray();
+                        imageBytes = imagePaths.Select(File.ReadAllBytes).ToList();
                     }
                     catch (IOException exception)
                     {
@@ -70,15 +67,17 @@ namespace LLama.Examples.Examples
                         break;
                     }
 
+                    // Each prompt with images we clear cache
+                    // When the prompt contains images we clear KV_CACHE to restart conversation
+                    // See:
+                    // https://github.com/ggerganov/llama.cpp/discussions/3620
+                    ex.Context.NativeHandle.KvCacheRemove( LLamaSeqId.Zero, -1, -1 );
 
                     int index = 0;
                     foreach (var path in imagePathsWithCurlyBraces)
                     {
                         // First image replace to tag <image, the rest of the images delete the tag
-                        if (index++ == 0)
-                            prompt = prompt.Replace(path, "<image>");
-                        else
-                            prompt = prompt.Replace(path, "");
+                        prompt = prompt.Replace(path, index++ == 0 ? "<image>" : "");
                     }
 
 
@@ -101,7 +100,10 @@ namespace LLama.Examples.Examples
 
                     // Initilize Images in executor
                     //
-                    ex.ImagePaths = imagePaths.ToList();
+                    foreach (var image in imagePaths)
+                    {
+                        ex.Images.Add(await File.ReadAllBytesAsync(image));
+                    }
                 }
 
                 Console.ForegroundColor = Color.White;
@@ -116,7 +118,7 @@ namespace LLama.Examples.Examples
 
                 // let the user finish with exit
                 //
-                if (prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase))
+                if (prompt != null && prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase))
                     break;
 
             }

docs/Tutorials/Executors.md

@@ -28,9 +28,9 @@ public interface ILLamaExecutor
     public LLavaWeights? ClipModel { get;  }        
 
     /// <summary>
-    /// List of images: Image filename and path (jpeg images).
+    /// List of images: List of images in byte array format.
     /// </summary>
-    public List<string> ImagePaths { get; set; }
+    public List<byte[]> Images { get; }
 
 
     /// <summary>