Rebased onto master and fixed breakage due to changes in `BaseSamplin…

…gPipeline`
SciSharp · Feb 25, 2024 · 879c62e · 879c62e
1 parent 8526d6b
commit 879c62e
Showing 1 changed file with 11 additions and 12 deletions.
diff --git a/LLama.Examples/Examples/BatchedExecutorGuidance.cs b/LLama.Examples/Examples/BatchedExecutorGuidance.cs
@@ -97,21 +97,20 @@ public override ISamplingPipeline Clone()
             throw new NotSupportedException();
         }
 
-        protected override IReadOnlyList<LLamaToken> GetProtectedTokens(SafeLLamaContextHandle ctx)
+        protected override ReadOnlySpan<float> ProcessLogits(SafeLLamaContextHandle ctx, ReadOnlySpan<float> logits, ReadOnlySpan<LLamaToken> lastTokens)
         {
-            return Array.Empty<LLamaToken>();
-        }
+            if (guidance == null)
+                return logits;
 
-        protected override void ProcessLogits(SafeLLamaContextHandle ctx, Span<float> logits, ReadOnlySpan<LLamaToken> lastTokens)
-        {
-            if (guidance != null)
-            {
-                // Get the logits generated by the guidance sequences
-                var guidanceLogits = guidance.Sample();
+            var logitsCopy = logits.ToArray();
+
+            // Get the logits generated by the guidance sequences
+            var guidanceLogits = guidance.Sample();
+
+            // Use those logits to guide this sequence
+            NativeApi.llama_sample_apply_guidance(ctx, logitsCopy, guidanceLogits, 2);
 
-                // Use those logits to guide this sequence
-                NativeApi.llama_sample_apply_guidance(ctx, logits, guidanceLogits, 2);
-            }
+            return logitsCopy;
         }
 
         protected override LLamaToken ProcessTokenDataArray(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan<LLamaToken> lastTokens)