diff --git a/LLama.Examples/Examples/BatchedExecutorGuidance.cs b/LLama.Examples/Examples/BatchedExecutorGuidance.cs index b1a13687c..12c22a1b8 100644 --- a/LLama.Examples/Examples/BatchedExecutorGuidance.cs +++ b/LLama.Examples/Examples/BatchedExecutorGuidance.cs @@ -97,21 +97,20 @@ public override ISamplingPipeline Clone() throw new NotSupportedException(); } - protected override IReadOnlyList GetProtectedTokens(SafeLLamaContextHandle ctx) + protected override ReadOnlySpan ProcessLogits(SafeLLamaContextHandle ctx, ReadOnlySpan logits, ReadOnlySpan lastTokens) { - return Array.Empty(); - } + if (guidance == null) + return logits; - protected override void ProcessLogits(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens) - { - if (guidance != null) - { - // Get the logits generated by the guidance sequences - var guidanceLogits = guidance.Sample(); + var logitsCopy = logits.ToArray(); + + // Get the logits generated by the guidance sequences + var guidanceLogits = guidance.Sample(); + + // Use those logits to guide this sequence + NativeApi.llama_sample_apply_guidance(ctx, logitsCopy, guidanceLogits, 2); - // Use those logits to guide this sequence - NativeApi.llama_sample_apply_guidance(ctx, logits, guidanceLogits, 2); - } + return logitsCopy; } protected override LLamaToken ProcessTokenDataArray(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan lastTokens)