diff --git a/LLama/Batched/Conversation.cs b/LLama/Batched/Conversation.cs index bcd371b00..c9a374549 100644 --- a/LLama/Batched/Conversation.cs +++ b/LLama/Batched/Conversation.cs @@ -135,9 +135,20 @@ public Conversation Fork() #region sample /// - /// Get the logits from this conversation, ready for sampling + /// Get the index in the context which each token can be sampled from, the return value of this function get be used to retrieve logits + /// () or to sample a token (. /// - /// How far from the end of the previous prompt should logits be sampled. Any value other than 0 requires allLogits to have been set during prompting + /// How far from the end of the previous prompt should logits be sampled. Any value other than 0 requires + /// allLogits to have been set during prompting.
+ /// For example if 5 tokens were supplied in the last prompt call: + /// + /// The logits of the first token can be accessed with 4 + /// The logits of the second token can be accessed with 3 + /// The logits of the third token can be accessed with 2 + /// The logits of the fourth token can be accessed with 1 + /// The logits of the fifth token can be accessed with 0 + /// + /// /// /// /// Thrown if this conversation was not prompted before the previous call to infer diff --git a/LLama/Batched/ConversationExtensions.cs b/LLama/Batched/ConversationExtensions.cs index aa56e8fb5..b6b0d9eb1 100644 --- a/LLama/Batched/ConversationExtensions.cs +++ b/LLama/Batched/ConversationExtensions.cs @@ -1,4 +1,5 @@ -using System; +using System; +using LLama.Native; namespace LLama.Batched; @@ -7,6 +8,18 @@ namespace LLama.Batched; /// public static class ConversationExtensions { + /// + /// Sample a token from this conversation using the given sampler chain + /// + /// to sample from + /// + /// Offset from the end of the conversation to the logits to sample, see for more details + /// + public static LLamaToken Sample(this Conversation conversation, SafeLLamaSamplerChainHandle sampler, int offset = 0) + { + return sampler.Sample(conversation.Executor.Context.NativeHandle, conversation.GetSampleIndex(offset)); + } + /// /// Rewind a back to an earlier state by removing tokens from the end ///