### Basic Semantic Kernel Setup with LLamaSharp

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/SemanticKernelPrompt

Initializes Semantic Kernel with LLamaSharp's text completion service. This setup is required before invoking any prompts.

```csharp
using LLama.Common;
using LLamaSharp.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel;
using LLamaSharp.SemanticKernel.TextCompletion;
using Microsoft.SemanticKernel.TextGeneration;
using Microsoft.Extensions.DependencyInjection;

namespace LLama.Examples.Examples
{
    // The basic example for using the semantic-kernel integration
    public class SemanticKernelPrompt
    {
        public static async Task Run()
        {
            string modelPath = UserSettings.GetModelPath();

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("This example is from: " +
                "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/README.md");

            // Load weights into memory
            var parameters = new ModelParams(modelPath);
            using var model = LLamaWeights.LoadFromFile(parameters);
            var ex = new StatelessExecutor(model, parameters);

            var builder = Kernel.CreateBuilder();
            builder.Services.AddKeyedSingleton<ITextGenerationService>("local-llama", new LLamaSharpTextCompletion(ex));

            var kernel = builder.Build();

            var prompt = @"{{$input}}

One line TLDR with the fewest words.";

            ChatRequestSettings settings = new() { MaxTokens = 100 };
            var summarize = kernel.CreateFunctionFromPrompt(prompt, settings);

            string text1 = @"
1st Law of Thermodynamics - Energy cannot be created or destroyed.
2nd Law of Thermodynamics - For a spontaneous process, the entropy of the universe increases.
3rd Law of Thermodynamics - A perfect crystal at zero Kelvin has zero entropy.";

            string text2 = @"
1. An object at rest remains at rest, and an object in motion remains in motion at constant speed and in a straight line unless acted on by an unbalanced force.
2. The acceleration of an object depends on the mass of the object and the amount of force applied.
3. Whenever one object exerts a force on another object, the second object exerts an equal and opposite on the first.";

            Console.WriteLine((await kernel.InvokeAsync(summarize, new() { ["input"] = text1 })).GetValue<string>());

            Console.WriteLine((await kernel.InvokeAsync(summarize, new() { ["input"] = text2 })).GetValue<string>());
        }
    }
}
```

--------------------------------

### StatelessExecutor Example Prompt

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Tutorials/Executors

Shows an example prompt for the StatelessExecutor, highlighting the need for a complete prompt as it lacks conversational memory.

```text
Q: Who is Trump? A:
```

--------------------------------

### Quantize Model Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/QuantizeModel

This example demonstrates how to quantize a model using the LLamaQuantizer class. It prompts the user for input and output paths, and the desired quantization type. Ensure you have the necessary model files and specify valid paths.

```csharp
namespace LLama.Examples.Examples
{
    public class QuantizeModel
    {
        public static void Run()
        {
            string inputPath = UserSettings.GetModelPath();

            Console.Write("Please input your output model path: ");
            var outputPath = Console.ReadLine();

            Console.Write("Please input the quantize type (one of q4_0, q4_1, q5_0, q5_1, q8_0): ");
            var quantizeType = Console.ReadLine();

            if (LLamaQuantizer.Quantize(inputPath, outputPath, quantizeType))
            {
                Console.WriteLine("Quantization succeeded!");
            }
            else
            {
                Console.WriteLine("Quantization failed!");
            }
        }
    }
}
```

--------------------------------

### Stateless Executor Basic Usage

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/StatelessModeExecute

This C# code demonstrates the fundamental setup and usage of the StatelessExecutor. It loads a model, initializes the executor, and enters a loop to process user questions as one-time inference tasks. Note that this example uses a minimal prompt and max tokens, which may affect performance.

```csharp
using LLama.Common;
using LLama.Examples.Extensions;

namespace LLama.Examples.Examples
{
    // Basic usage of the stateless executor.
    public class StatelessModeExecute
    {
        public static async Task Run()
        {
            string modelPath = UserSettings.GetModelPath();

            var parameters = new ModelParams(modelPath)
            {
                ContextSize = 1024,
                Seed = 1337,
                GpuLayerCount = 5
            };
            using var model = LLamaWeights.LoadFromFile(parameters);
            var ex = new StatelessExecutor(model, parameters);

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("The executor has been enabled. In this example, the inference is an one-time job. That says, the previous input and response has " +
                "no impact on the current response. Now you can ask it questions. Note that in this example, no prompt was set for LLM and the maximum response tokens is 50. " +
                "It may not perform well because of lack of prompt. This is also an example that could indicate the importance of prompt in LLM. To improve it, you can add " +
                "a prompt for it yourself!");
            Console.ForegroundColor = ConsoleColor.White;

            var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "Question:", "#", "Question: ", ".\n" }, MaxTokens = 50 };

            while (true)
            {
                Console.Write("\nQuestion: ");
                Console.ForegroundColor = ConsoleColor.Green;
                var prompt = Console.ReadLine();
                Console.ForegroundColor = ConsoleColor.White;
                Console.Write("Answer: ");
                prompt = $"Question: {prompt?.Trim()} Answer: ";
                await foreach (var text in ex.InferAsync(prompt, inferenceParams).Spinner())
                {
                    Console.Write(text);
                }
            }
        }
    }
}
```

--------------------------------

### Example Prompt for Anti-Prompt Configuration

Source: https://scisharp.github.io/LLamaSharp/0.25.0/FAQ

This example shows a transcript of a dialog. The anti-prompt should be set to 'User:' to prevent infinite output generation, especially in interactive modes where max-length is not set.

```text
1
2
3
4
5
6
7

Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.

User: Hello, Bob.
Bob: Hello. How may I help you today?
User: Please tell me the largest city in Europe.
Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
User:
```

--------------------------------

### BatchedExecutor for Guided Generation

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Tutorials/Executors

This example shows how to use BatchedExecutor to generate text with and without classifier-free guidance. It sets up two conversations, one for positive prompting and one for negative prompting, and uses a GuidedSampler to steer the generation process.

```csharp
using LLama.Batched;
using LLama.Common;
using LLama.Native;
using LLama.Sampling;
using Spectre.Console;

namespace LLama.Examples.Examples;

/// <summary>
/// This demonstrates using a batch to generate two sequences and then using one
/// sequence as the negative guidance ("classifier free guidance") for the other.
/// </summary>
public class BatchedExecutorGuidance
{
    private const int n_len = 32;

    public static async Task Run()
    {
        string modelPath = UserSettings.GetModelPath();

        var parameters = new ModelParams(modelPath);
        using var model = LLamaWeights.LoadFromFile(parameters);

        var positivePrompt = AnsiConsole.Ask("Positive Prompt (or ENTER for default):", "My favourite colour is").Trim();
        var negativePrompt = AnsiConsole.Ask("Negative Prompt (or ENTER for default):", "I hate the colour red. My favourite colour is").Trim();
        var weight = AnsiConsole.Ask("Guidance Weight (or ENTER for default):", 2.0f);

        // Create an executor that can evaluate a batch of conversations together
        using var executor = new BatchedExecutor(model, parameters);

        // Print some info
        var name = executor.Model.Metadata.GetValueOrDefault("general.name", "unknown model name");
        Console.WriteLine($"Created executor with model: {name}");

        // Load the two prompts into two conversations
        using var guided = executor.Create();
        guided.Prompt(positivePrompt);
        using var guidance = executor.Create();
        guidance.Prompt(negativePrompt);

        // Run inference to evaluate prompts
        await AnsiConsole
             .Status()
             .Spinner(Spinner.Known.Line)
             .StartAsync("Evaluating Prompts...", _ => executor.Infer());

        // Fork the "guided" conversation. We'll run this one without guidance for comparison
        using var unguided = guided.Fork();

        // Run inference loop
        var unguidedSampler = new GuidedSampler(null, weight);
        var unguidedDecoder = new StreamingTokenDecoder(executor.Context);
        var guidedSampler = new GuidedSampler(guidance, weight);
        var guidedDecoder = new StreamingTokenDecoder(executor.Context);
        await AnsiConsole
           .Progress()
           .StartAsync(async progress =>
            {
                var reporter = progress.AddTask("Running Inference", maxValue: n_len);

                for (var i = 0; i < n_len; i++)
                {
                    if (i != 0)
                        await executor.Infer();

                    // Sample from the "unguided" conversation. This is just a conversation using the same prompt, without any
                    // guidance. This serves as a comparison to show the effect of guidance.
                    var u = unguidedSampler.Sample(executor.Context.NativeHandle, unguided.Sample(), Array.Empty<LLamaToken>());
                    unguidedDecoder.Add(u);
                    unguided.Prompt(u);

                    // Sample from the "guided" conversation. This sampler will internally use the "guidance" conversation
                    // to steer the conversation. See how this is done in GuidedSampler.ProcessLogits (bottom of this file).
                    var g = guidedSampler.Sample(executor.Context.NativeHandle, guided.Sample(), Array.Empty<LLamaToken>());
                    guidedDecoder.Add(g);

                    // Use this token to advance both guided _and_ guidance. Keeping them in sync (except for the initial prompt).
                    guided.Prompt(g);
                    guidance.Prompt(g);

                    // Early exit if we reach the natural end of the guided sentence
                    if (g == model.EndOfSentenceToken)
                        break;

                    // Update progress bar
                    reporter.Increment(1);
                }
            });

        AnsiConsole.MarkupLine($"[green]Unguided:[/][white]{unguidedDecoder.Read().ReplaceLineEndings(" ")}[/]");
        AnsiConsole.MarkupLine($"[green]Guided:[/][white]{guidedDecoder.Read().ReplaceLineEndings(" ")}[/]");
    }

    private class GuidedSampler(Conversation? guidance, float weight)
        : BaseSamplingPipeline
    {

```

```csharp
private static void ProcessLogits(IntPtr nativeHandle, Span<float> logits, Conversation? guidance, float weight)
    {
        if (guidance == null || weight <= 0.0f)
        {
            return; // No guidance to apply
        }

        // Get the logits for the guidance sequence
        var guidanceLogits = LLamaNative.llama_sample_token_greedy(nativeHandle, guidance.Sample().NativeHandle);

        // Apply guidance: subtract the guidance logits scaled by the weight
        for (var i = 0; i < logits.Length; i++)
        {
            logits[i] = logits[i] - guidanceLogits[i] * weight;
        }
    }

    public override LLamaToken Sample(IntPtr nativeHandle, LLamaTokenDataArray sampled, ReadOnlySpan<LLamaToken> lastTokens)
    {
        // Get the current logits from the context
        var logits = LLamaNative.llama_get_logits(nativeHandle);

        // Apply guidance
        ProcessLogits(nativeHandle, logits, guidance, weight);

        // Now sample from the modified logits
        return base.Sample(nativeHandle, sampled, lastTokens);
    }
}

```

--------------------------------

### Install LLamaSharp Package

Source: https://scisharp.github.io/LLamaSharp/0.25.0/QuickStart

Install the LLamaSharp NuGet package using the Package Manager Console.

```powershell
PM> Install-Package LLamaSharp
```

--------------------------------

### InstructExecutor Prompt Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Tutorials/Executors

Illustrates the prompt format for the InstructExecutor, which is designed to follow specific instructions provided by the user.

```text
// alpaca

Below is an instruction that describes a task. Write a response that appropriately completes the request.


```

--------------------------------

### Sample Token Using SafeLLamaSamplerChainHandle

Source: https://scisharp.github.io/LLamaSharp/0.25.0/xmldocs/llama.native.safellamasamplerchainhandle

Demonstrates how to sample and accept a token from the LLama context using a sampler chain. This involves getting logits, creating a token data array, applying the sampler chain, and accepting the selected token.

```csharp
var logits = ctx.GetLogitsIth(idx);
var token_data_array = LLamaTokenDataArray.Create(logits);
using LLamaTokenDataArrayNative.Create(token_data_array, out var native_token_data);
sampler_chain.Apply(native_token_data);
var token = native_token_data.Data.Span[native_token_data.Selected];
sampler_chain.Accept(token);
return token;
```

```csharp
public LLamaToken Sample(SafeLLamaContextHandle context, int index)
```

--------------------------------

### Coding Assistant with LLamaSharp

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/CodingAssistant

This C# code demonstrates setting up and running a coding assistant using LLamaSharp. It initializes the model and executor, then enters a loop to accept user instructions and generate code completions. It's recommended to use a Code Llama model for this example.

```csharp
namespace LLama.Examples.Examples
{
    using LLama.Common;
    using System;

    // This example shows how to apply code completion as a coding assistant
    internal class CodingAssistant
    {
        // Source paper with example prompts:
        // https://doi.org/10.48550/arXiv.2308.12950
        const string InstructionPrefix = "[INST]";
        const string InstructionSuffix = "[/INST]";
        const string SystemInstruction = "You're an intelligent, concise coding assistant. " +
            "Wrap code in ``` for readability. Don't repeat yourself. " +
            "Use best practice and good coding standards.";

        public static async Task Run()
        {
            string modelPath = UserSettings.GetModelPath();
            if (!modelPath.Contains("codellama", StringComparison.InvariantCultureIgnoreCase))
            {
                Console.ForegroundColor = ConsoleColor.Yellow;
                Console.WriteLine("WARNING: the model you selected is not a Code LLama model!");
                Console.WriteLine("For this example we specifically recommend 'codellama-7b-instruct.Q4_K_S.gguf'");
                Console.WriteLine("Press ENTER to continue...");
                Console.ReadLine();
            }

            var parameters = new ModelParams(modelPath)
            {
                ContextSize = 4096
            };
            using var model = LLamaWeights.LoadFromFile(parameters);
            using var context = model.CreateContext(parameters);
            var executor = new InstructExecutor(context, InstructionPrefix, InstructionSuffix, null);

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions." +
                "\nIt's a 7B Code Llama, so it's trained for programming tasks like \"Write a C# function reading " +
                "a file name from a given URI\" or \"Write some programming interview questions\"." +
                "\nWrite 'exit' to exit");
            Console.ForegroundColor = ConsoleColor.White;

            var inferenceParams = new InferenceParams()
            {
                Temperature = 0.8f,
                MaxTokens = -1,
            };

            string instruction = $"{SystemInstruction}\n\n";
            await Console.Out.WriteAsync("Instruction: ");
            instruction += Console.ReadLine() ?? "Ask me for instructions.";
            while (instruction != "exit")
            {

                Console.ForegroundColor = ConsoleColor.Green;
                await foreach (var text in executor.InferAsync(instruction + Environment.NewLine, inferenceParams))
                {
                    Console.Write(text);
                }
                Console.ForegroundColor = ConsoleColor.White;

                await Console.Out.WriteAsync("Instruction: ");
                instruction = Console.ReadLine() ?? "Ask me for instructions.";
            }
        }
    }
}

```

--------------------------------

### LLamaSharp Chat Session Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/QuickStart

Demonstrates a basic chat session with an LLM using LLamaSharp. Ensure you replace '<Your Model Path>' with the actual path to your GGUF model file. Adjust GpuLayerCount based on your GPU memory.

```csharp
using LLama.Common;
using LLama;

string modelPath = @"<Your Model Path>"; // change it to your own model path. 

var parameters = new ModelParams(modelPath)
{
    ContextSize = 1024, // The longest length of chat as memory.
    GpuLayerCount = 5 // How many layers to offload to GPU. Please adjust it according to your GPU memory.
};
using var model = LLamaWeights.LoadFromFile(parameters);
using var context = model.CreateContext(parameters);
var executor = new InteractiveExecutor(context);

// Add chat histories as prompt to tell AI how to act.
var chatHistory = new ChatHistory();
chatHistory.AddMessage(AuthorRole.System, "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.");
chatHistory.AddMessage(AuthorRole.User, "Hello, Bob.");
chatHistory.AddMessage(AuthorRole.Assistant, "Hello. How may I help you today?");

ChatSession session = new(executor, chatHistory);

InferenceParams inferenceParams = new InferenceParams()
{
    MaxTokens = 256, // No more than 256 tokens should appear in answer. Remove it if antiprompt is enough for control.
    AntiPrompts = new List<string> { "User:" } // Stop generation once antiprompts appear.
};

Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write("The chat session has started.\nUser: ");
Console.ForegroundColor = ConsoleColor.Green;
string userInput = Console.ReadLine() ?? "";

while (userInput != "exit")
{
    await foreach (
        var text
        in session.ChatAsync(
            new ChatHistory.Message(AuthorRole.User, userInput),
            inferenceParams))
    {
        Console.ForegroundColor = ConsoleColor.White;
        Console.Write(text);
    }
    Console.ForegroundColor = ConsoleColor.Green;
    userInput = Console.ReadLine() ?? "";
}

```

--------------------------------

### Chinese Chat with GB2312 Encoding Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/ChatChineseGB2312

This C# example demonstrates how to initialize and run a chat session with a Chinese language model using GB2312 encoding. It includes setting up the model parameters, loading a chat session from disk or JSON, and handling user input and model responses with necessary encoding conversions.

```csharp
using System.Text;
using LLama.Common;

namespace LLama.Examples.Examples;

// This example shows how to deal with Chinese input with gb2312 encoding.
public class ChatChineseGB2312
{
    private static string ConvertEncoding(string input, Encoding original, Encoding target)
    {
        byte[] bytes = original.GetBytes(input);
        var convertedBytes = Encoding.Convert(original, target, bytes);
        return target.GetString(convertedBytes);
    }

    public static async Task Run()
    {
        // Register provider for GB2312 encoding
        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);

        Console.ForegroundColor = ConsoleColor.Yellow;
        Console.WriteLine("This example shows how to use Chinese with gb2312 encoding, which is common in windows. It's recommended" +
            " to use https://huggingface.co/hfl/chinese-alpaca-2-7b-gguf/blob/main/ggml-model-q5_0.gguf, which has been verified by LLamaSharp developers.");
        Console.ForegroundColor = ConsoleColor.White;

        string modelPath = UserSettings.GetModelPath();

        var parameters = new ModelParams(modelPath)
        {
            ContextSize = 1024,
            Seed = 1337,
            GpuLayerCount = 5,
            Encoding = Encoding.UTF8
        };
        using var model = LLamaWeights.LoadFromFile(parameters);
        using var context = model.CreateContext(parameters);
        var executor = new InteractiveExecutor(context);

        ChatSession session;
        if (Directory.Exists("Assets/chat-with-kunkun-chinese"))
        {
            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("Loading session from disk.");
            Console.ForegroundColor = ConsoleColor.White;

            session = new ChatSession(executor);
            session.LoadSession("Assets/chat-with-kunkun-chinese");
        }
        else
        {
            var chatHistoryJson = File.ReadAllText("Assets/chat-with-kunkun-chinese.json");
            ChatHistory chatHistory = ChatHistory.FromJson(chatHistoryJson) ?? new ChatHistory();

            session = new ChatSession(executor, chatHistory);
        }

        session
            .WithHistoryTransform(new LLamaTransforms.DefaultHistoryTransform("用户", "坤坤"));

        InferenceParams inferenceParams = new InferenceParams()
        {
            Temperature = 0.9f,
            AntiPrompts = new List<string> { "用户：" }
        };

        Console.ForegroundColor = ConsoleColor.Yellow;
        Console.WriteLine("The chat session has started.");

        // show the prompt
        Console.ForegroundColor = ConsoleColor.White;
        Console.Write("用户：");
        Console.ForegroundColor = ConsoleColor.Green;
        string userInput = Console.ReadLine() ?? "";

        while (userInput != "exit")
        {
            // Convert the encoding from gb2312 to utf8 for the language model
            // and later saving to the history json file.
            userInput = ConvertEncoding(userInput, Encoding.GetEncoding("gb2312"), Encoding.UTF8);

            if (userInput == "save")
            {
                session.SaveSession("Assets/chat-with-kunkun-chinese");
                Console.ForegroundColor = ConsoleColor.Yellow;
                Console.WriteLine("Session saved.");
            }
            else if (userInput == "regenerate")
            {
                Console.ForegroundColor = ConsoleColor.Yellow;
                Console.WriteLine("Regenerating last response ...");

                await foreach (
                    var text
                    in session.RegenerateAssistantMessageAsync(
                        inferenceParams))
                {
                    Console.ForegroundColor = ConsoleColor.White;

                    // Convert the encoding from utf8 to gb2312 for the console output.
                    Console.Write(ConvertEncoding(text, Encoding.UTF8, Encoding.GetEncoding("gb2312")));
                }
            }
            else
            {
                await foreach (
                    var text
                    in session.ChatAsync(
                        new ChatHistory.Message(AuthorRole.User, userInput),
                        inferenceParams))
                {
                    Console.ForegroundColor = ConsoleColor.White;
                    Console.Write(text);
                }
            }

            Console.ForegroundColor = ConsoleColor.Green;

```

--------------------------------

### Basic Instruct Mode Execution in C#

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/InstructModeExecute

Initializes the InstructExecutor with a model and context, then enters a loop to continuously process user prompts and generate responses. Requires model path and prompt file setup.

```csharp
using LLama.Common;

namespace LLama.Examples.Examples
{
    // This example shows how to use InstructExecutor to generate the response.
    public class InstructModeExecute
    {
        public static async Task Run()
        {
            string modelPath = UserSettings.GetModelPath();

            var prompt = File.ReadAllText("Assets/dan.txt").Trim();

            var parameters = new ModelParams(modelPath)
            {
                ContextSize = 1024,
                Seed = 1337,
                GpuLayerCount = 5
            };
            using var model = LLamaWeights.LoadFromFile(parameters);
            using var context = model.CreateContext(parameters);
            var executor = new InstructExecutor(context);

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions. For example, you can input \"Write a story about a fox who want to " + 
                "make friend with human, no less than 200 words.\"");
            Console.ForegroundColor = ConsoleColor.White;

            var inferenceParams = new InferenceParams() { Temperature = 0.8f, MaxTokens = 600 };

            while (true)
            {
                await foreach (var text in executor.InferAsync(prompt, inferenceParams))
                {
                    Console.Write(text);
                }
                Console.ForegroundColor = ConsoleColor.Green;
                prompt = Console.ReadLine();
                Console.ForegroundColor = ConsoleColor.White;
            }
        }
    }
}
```

--------------------------------

### Run Kernel Memory Save and Load Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/KernelMemorySaveAndLoad

Orchestrates the Kernel Memory save and load process. It checks for existing storage, ingests documents if necessary, and then allows for querying.

```csharp
public static async Task Run()
    {
        Console.ForegroundColor = ConsoleColor.Yellow;
        Console.WriteLine(
            """

            This program uses the Microsoft.KernelMemory package to ingest documents
            and store the embeddings as local files so they can be quickly recalled
            when this application is launched again. 

            """);

        string modelPath = UserSettings.GetModelPath();
        IKernelMemory memory = CreateMemoryWithLocalStorage(modelPath);

        Console.ForegroundColor = ConsoleColor.Yellow;
        if (StorageExists)
        {
            Console.WriteLine(
                """

                Kernel memory files have been located!
                Information about previously analyzed documents has been loaded.

                """);
        }
        else
        {
            Console.WriteLine(
                $"""

                 Existing kernel memory was not found.
                 Documents will be analyzed (slow) and information saved to disk.
                 Analysis will not be required the next time this program is run.
                 Press ENTER to proceed...

                 """);
            Console.ReadLine();
            await IngestDocuments(memory);
        }

        await AskSingleQuestion(memory, "What formats does KM support?");
        await StartUserChatSession(memory);
    }
```

--------------------------------

### Ask a Question and Get an Answer

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/KernelMemorySaveAndLoad

Queries the kernel memory with a given question and displays the answer along with relevant sources. The time taken for answer generation is also logged.

```csharp
private static async Task ShowAnswer(IKernelMemory memory, string question)
    {
        Stopwatch sw = Stopwatch.StartNew();
        Console.ForegroundColor = ConsoleColor.DarkGray;
        Console.WriteLine($"Generating answer...");

        MemoryAnswer answer = await memory.AskAsync(question);
        Console.WriteLine($"Answer generated in {sw.Elapsed}");

        Console.ForegroundColor = ConsoleColor.Gray;
        Console.WriteLine($"Answer: {answer.Result}");
        foreach (var source in answer.RelevantSources)
        {
            Console.WriteLine($"Source: {source.SourceName}");
        }
        Console.WriteLine();
    }
```

--------------------------------

### StatelessExecutor Response Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Tutorials/Executors

Provides a sample response from the StatelessExecutor when given a prompt like 'Q: Who is Trump? A:', demonstrating its direct answer capability without prior context.

```text
Donald J. Trump, born June 14, 1946, is an American businessman, television personality, politician and the 45th President of the United States (2017-2021). # Anexo:Torneo de Hamburgo 2022 (individual masculino)
```

--------------------------------

### InteractiveExecutor Chat Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Tutorials/Executors

Demonstrates a typical conversational prompt structure for the InteractiveExecutor, simulating a user interacting with an assistant named Bob.

```text
// chat-with-bob

Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.

User: Hello, Bob.
Bob: Hello. How may I help you today?
User: Please tell me the largest city in Europe.
Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
User:
```

--------------------------------

### Install xmldoc2md Tool

Source: https://scisharp.github.io/LLamaSharp/0.25.0/ContributingGuide

Installs the XMLDoc2Markdown tool globally using the .NET CLI. This tool is used for generating markdown documentation from XML documentation comments.

```bash
dotnet tool install -g XMLDoc2Markdown
```

--------------------------------

### HelpLink Property

Source: https://scisharp.github.io/LLamaSharp/0.25.0/xmldocs/llama.batched.cannotsavewhilerequiresinferenceexception

Gets or sets a link to the help file associated with this exception. This is a standard Exception property.

```csharp
public string HelpLink { get; set; }
```

--------------------------------

### BatchedExecutor Fork Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/BatchedExecutorFork

Demonstrates splitting a sequence into multiple forked sequences using BatchedExecutor for simultaneous inference. This method shares memory up to the split point, optimizing resource usage.

```csharp
using var executor = new BatchedExecutor(model, "Not many people know that");
var fork1 = executor.Fork();
var fork2 = executor.Fork();

await Parallel.WhenAll(fork1.GenerateAsync(), fork2.GenerateAsync());

Console.WriteLine("Fork 1 completed.");
Console.WriteLine("Fork 2 completed.");

Console.WriteLine("Fork 1 output:");
Console.WriteLine(await fork1.GetResultAsync());

Console.WriteLine("Fork 2 output:");
Console.WriteLine(await fork2.GetResultAsync());
```

--------------------------------

### Create Kernel Memory with Local Storage

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/KernelMemorySaveAndLoad

Configures and builds an IKernelMemory instance using local file storage for both document content and vector embeddings. This setup allows for persistence and faster loading on subsequent runs.

```csharp
private static IKernelMemory CreateMemoryWithLocalStorage(string modelPath)
    {
        Common.InferenceParams infParams = new() { AntiPrompts = ["\n\n"] };

        LLamaSharpConfig lsConfig = new(modelPath) { DefaultInferenceParams = infParams };

        SearchClientConfig searchClientConfig = new() 
        {
            MaxMatchesCount = 1,
            AnswerTokens = 100,
        };

        TextPartitioningOptions parseOptions = new() 
        {
            MaxTokensPerParagraph = 300,
            MaxTokensPerLine = 100,
            OverlappingTokens = 30
        };

        SimpleFileStorageConfig storageConfig = new() 
        {
            Directory = StorageFolder,
            StorageType = FileSystemTypes.Disk,
        };

        SimpleVectorDbConfig vectorDbConfig = new() 
        {
            Directory = StorageFolder,
            StorageType = FileSystemTypes.Disk,
        };

        Console.ForegroundColor = ConsoleColor.Blue;
        Console.WriteLine($"Kernel memory folder: {StorageFolder}");

        Console.ForegroundColor = ConsoleColor.DarkGray);
        return new KernelMemoryBuilder()
            .WithSimpleFileStorage(storageConfig)
            .WithSimpleVectorDb(vectorDbConfig)
            .WithLLamaSharpDefaults(lsConfig)
            .WithSearchClientConfig(searchClientConfig)
            .With(parseOptions)
            .Build();
    }
```

--------------------------------

### LLamaSharp Chat Completion Example

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/SemanticKernelChat

This C# code demonstrates setting up and using LLamaSharp's chat completion feature with Semantic Kernel. It initializes a chat history, adds user messages, and retrieves AI-generated responses, simulating a conversation with a librarian.

```csharp
using LLama.Common;
using LLamaSharp.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.ChatCompletion;

namespace LLama.Examples.Examples
{
    public class SemanticKernelChat
    {
        public static async Task Run()
        {
            string modelPath = UserSettings.GetModelPath();

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("This example is from: \n" +
                "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/KernelSyntaxExamples/Example17_ChatGPT.cs");

            // Load weights into memory
            var parameters = new ModelParams(modelPath);
            using var model = LLamaWeights.LoadFromFile(parameters);
            var ex = new StatelessExecutor(model, parameters);

            var chatGPT = new LLamaSharpChatCompletion(ex);

            var chatHistory = chatGPT.CreateNewChat("This is a conversation between the " +
                "assistant and the user. \n\n You are a librarian, expert about books. ");

            Console.WriteLine("Chat content:");
            Console.WriteLine("------------------------");

            chatHistory.AddUserMessage("Hi, I'm looking for book suggestions");
            await MessageOutputAsync(chatHistory);

            // First bot assistant message
            var reply = await chatGPT.GetChatMessageContentAsync(chatHistory);
            chatHistory.AddAssistantMessage(reply.Content);
            await MessageOutputAsync(chatHistory);

            // Second user message
            chatHistory.AddUserMessage("I love history and philosophy, I'd like to learn " +
                "something new about Greece, any suggestion");
            await MessageOutputAsync(chatHistory);

            // Second bot assistant message
            reply = await chatGPT.GetChatMessageContentAsync(chatHistory);
            chatHistory.AddAssistantMessage(reply.Content);
            await MessageOutputAsync(chatHistory);
        }

        /// <summary>
        /// Outputs the last message of the chat history
        /// </summary>
        private static Task MessageOutputAsync(Microsoft.SemanticKernel.ChatCompletion.ChatHistory chatHistory)
        {
            var message = chatHistory.Last();

            Console.WriteLine($"{message.Role}: {message.Content}");
            Console.WriteLine("------------------------");

            return Task.CompletedTask;
        }
    }
}
```

--------------------------------

### Start User Chat Session

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/KernelMemorySaveAndLoad

Initiates an interactive chat session where users can ask questions continuously. The session ends when the user enters an empty line.

```csharp
private static async Task StartUserChatSession(IKernelMemory memory)
    {
        while (true)
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.Write("Question: ");
            string question = Console.ReadLine()!;
            if (string.IsNullOrEmpty(question))
                return;

            await ShowAnswer(memory, question);
        }
    }
```

--------------------------------

### Add ITextEmbeddingGeneration Service to Semantic Kernel

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Integrations/semantic-kernel

Set up Semantic Kernel to generate text embeddings using a local LLaMa model. This example also configures a volatile memory store.

```csharp
using var model = LLamaWeights.LoadFromFile(parameters);
var embedding = new LLamaEmbedder(model, parameters);
var kernelWithCustomDb = Kernel.Builder
    .WithLoggerFactory(ConsoleLogger.LoggerFactory)
    .WithAIService<ITextEmbeddingGeneration>("local-llama-embed", new LLamaSharpEmbeddingGeneration(embedding), true)
    .WithMemoryStorage(new VolatileMemoryStore())
    .Build();
```

--------------------------------

### JSON Response Generation with Grammar

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/GrammarJsonResponse

This C# code snippet demonstrates how to use LLamaSharp to generate responses in a JSON format by employing a grammar. It sets up the model, loads a GBNF grammar file, and configures the inference parameters to enforce the grammar. The example runs in a loop, prompting the user for input and displaying the LLM's JSON-formatted answer.

```csharp
using LLama.Common;
using LLama.Grammars;

namespace LLama.Examples.Examples
{
    // This example shows how to get response in json format using grammar.
    public class GrammarJsonResponse
    {
        public static async Task Run()
        {
            string modelPath = UserSettings.GetModelPath();

            var gbnf = File.ReadAllText("Assets/json.gbnf").Trim();
            var grammar = Grammar.Parse(gbnf, "root");

            var parameters = new ModelParams(modelPath)
            {
                ContextSize = 1024,
                Seed = 1337,
                GpuLayerCount = 5
            };
            using var model = LLamaWeights.LoadFromFile(parameters);
            var ex = new StatelessExecutor(model, parameters);

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions and always respond in a JSON format. For example, you can input \"Tell me the attributes of a good dish\"");
            Console.ForegroundColor = ConsoleColor.White;

            using var grammarInstance = grammar.CreateInstance();
            var inferenceParams = new InferenceParams()
            {
                Temperature = 0.6f,
                AntiPrompts = new List<string> { "Question:", "#", "Question: ", ".\n" },
                MaxTokens = 50,
                Grammar = grammarInstance
            };

            while (true)
            {
                Console.Write("\nQuestion: ");
                Console.ForegroundColor = ConsoleColor.Green;
                var prompt = Console.ReadLine();
                Console.ForegroundColor = ConsoleColor.White;
                Console.Write("Answer: ");
                prompt = $
```

--------------------------------

### Get Text Embeddings with LlamaSharp

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/GetEmbeddings

Use this code to generate embeddings for a given text prompt. Ensure the LlamaSharp library is installed and a model is available. The output will be an array of floating-point numbers representing the text.

```csharp
using LLama.Common;

namespace LLama.Examples.Examples
{
    // This example shows how to get embeddings from a text prompt.
    public class GetEmbeddings
    {
        public static void Run()
        {
            string modelPath = UserSettings.GetModelPath();

            Console.ForegroundColor = ConsoleColor.DarkGray;
            var @params = new ModelParams(modelPath) { EmbeddingMode = true };
            using var weights = LLamaWeights.LoadFromFile(@params);
            var embedder = new LLamaEmbedder(weights, @params);

            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine(
                """
                This example displays embeddings from a text prompt.
                Embeddings are numerical codes that represent information like words, images, or concepts.
                These codes capture important relationships between those objects, 
                like how similar words are in meaning or how close images are visually.
                This allows machine learning models to efficiently understand and process complex data.
                Embeddings of a text in LLM is sometimes useful, for example, to train other MLP models.
                """); // NOTE: this description was AI generated

            while (true)
            {
                Console.ForegroundColor = ConsoleColor.White;
                Console.Write("Please input your text: ");
                Console.ForegroundColor = ConsoleColor.Green;
                var text = Console.ReadLine();
                Console.ForegroundColor = ConsoleColor.White;

                float[] embeddings = embedder.GetEmbeddings(text).Result;
                Console.WriteLine($"Embeddings contain {embeddings.Length:N0} floating point values:");
                Console.ForegroundColor = ConsoleColor.DarkGray;
                Console.WriteLine(string.Join(", ", embeddings.Take(20)) + ", ");
                Console.WriteLine();
            }
        }
    }
}
```

--------------------------------

### Implement INativeLibrary Interface

Source: https://scisharp.github.io/LLamaSharp/0.25.0/AdvancedTutorials/CustomizeNativeLibraryLoading

Implement this interface to customize how a native library is prepared and loaded. The `Prepare` method allows for pre-loading actions like moving or downloading files.

```csharp
public interface INativeLibrary
{
    /// <summary>
    /// Metadata of this library.
    /// </summary>
    NativeLibraryMetadata? Metadata { get; }

    /// <summary>
    /// Prepare the native library file and returns the local path of it.
    /// If it's a relative path, LLamaSharp will search the path in the search directies you set.
    /// </summary>
    /// <param name="systemInfo">The system information of the current machine.</param>
    /// <param name="logCallback">The log callback.</param>
    /// <returns>
    /// The relative paths of the library. You could return multiple paths to try them one by one. If no file is available, please return an empty array.
    /// </returns>
    IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null);
}
```

--------------------------------

### Sample Data for Memory Storage

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/SemanticKernelMemory

Provides a dictionary of sample GitHub URLs and their descriptions to be used when populating the semantic memory.

```csharp
private static Dictionary<string, string> SampleData()
{
    return new Dictionary<string, string>
    {
        ["https://github.com/microsoft/semantic-kernel/blob/main/README.md"]
            = "README: Installation, getting started, and how to contribute",
        ["https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb"]
            = "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function",
        ["https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks//00-getting-started.ipynb"]
            = "Jupyter notebook describing how to get started with the Semantic Kernel",
        ["https://github.com/microsoft/semantic-kernel/blob/main/samples/skills/ChatSkill/ChatGPT"]
            = "Sample demonstrating how to create a chat skill interfacing with ChatGPT",
        ["https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs"]
            = "C# class that defines a volatile embedding store",
        ["https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md"]
            = "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4",
        ["https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md"]
            = "README: README associated with a sample chat summary react-based webapp",
    };
}
```

--------------------------------

### Get Text Embeddings with LLamaEmbedder

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Tutorials/GetEmbeddings

Initialize LLamaEmbedder with model parameters and then call GetEmbeddings to get a float array representing the text's embedding. The output array's length depends on the loaded model.

```csharp
var embedder = new LLamaEmbedder(new ModelParams("<modelPath>"));
string text = "hello, LLM.";
float[] embeddings = embedder.GetEmbeddings(text);
```

--------------------------------

### Images Property

Source: https://scisharp.github.io/LLamaSharp/0.25.0/xmldocs/llama.statelessexecutor

Gets the list of images associated with the current inference context.

```csharp
public List<Byte[]> Images { get; }
```

--------------------------------

### Implement Remote Native Library Downloading

Source: https://scisharp.github.io/LLamaSharp/0.25.0/AdvancedTutorials/CustomizeNativeLibraryLoading

This C# code defines a wrapper for CUDA native libraries that integrates with LLamaSharp's downloading mechanism. It allows for custom download settings and prepares libraries by attempting to load from default paths first, then triggering a download if necessary. Note that the `Prepare` method cannot be asynchronous.

```csharp
public class AutoDownloadedLibraries
{
    // Wrap a cuda native library
    public class Cuda: INativeLibrary
    {
        // the default cuda native library implementation in LLamaSHarp
        private NativeLibraryWithCuda _cudaLibrary;
        // Some download settings
        private NativeLibraryDownloadSettings _settings;

        public Cuda(NativeLibraryWithCuda cudaLibrary, NativeLibraryDownloadSettings settings)
        {
            _cudaLibrary = cudaLibrary;
            _settings = settings;
        }

        public NativeLibraryMetadata? Metadata => _cudaLibrary.Metadata;

        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null)
        {
            foreach(var relativePath in _cudaLibrary.Prepare(systemInfo, logCallback))
            {
                // try to use the default path first. If loaded successfully, the download will not be triggered.
                yield return relativePath;
                // download the file.
                // NOTE: be sure to complete the downloading process here. You CANNOT make `Prepare` as an async method.
                var path = NativeLibraryDownloader.DownloadLibraryFile(_settings, relativePath, logCallback).Result;
                // if the downloading is successful, return the path of the downloaded file.
                if (path is not null)
                {
                    yield return path;
                }
            }
        }
    }
}
```

--------------------------------

### Kernel Memory Integration with LLamaSharp

Source: https://scisharp.github.io/LLamaSharp/0.25.0/Examples/KernelMemory

Sets up Kernel Memory with LLamaSharp, ingests documents, and allows interactive question answering. Requires document files in an 'Assets' folder.

```csharp
using LLamaSharp.KernelMemory;
using Microsoft.KernelMemory;
using Microsoft.KernelMemory.Configuration;
using System.Diagnostics;

namespace LLama.Examples.Examples
{
    // This example is from Microsoft's official kernel memory "custom prompts" example:
    // https://github.com/microsoft/kernel-memory/blob/6d516d70a23d50c6cb982e822e6a3a9b2e899cfa/examples/101-dotnet-custom-Prompts/Program.cs#L1-L86

    // Microsoft.KernelMemory has more features than Microsoft.SemanticKernel.
    // See https://microsoft.github.io/kernel-memory/ for details.

    public class KernelMemory
    {
        public static async Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine(
                """

                This program uses the Microsoft.KernelMemory package to ingest documents
                and answer questions about them in an interactive chat prompt.

                """);

            // Setup the kernel memory with the LLM model
            string modelPath = UserSettings.GetModelPath();
            IKernelMemory memory = CreateMemory(modelPath);

            // Ingest documents (format is automatically detected from the filename)
            string[] filesToIngest = [
                Path.GetFullPath(@"./Assets/sample-SK-Readme.pdf"),
                Path.GetFullPath(@"./Assets/sample-KM-Readme.pdf"),
            ];

            for (int i = 0; i < filesToIngest.Length; i++)
            {
                string path = filesToIngest[i];
                Stopwatch sw = Stopwatch.StartNew();
                Console.ForegroundColor = ConsoleColor.Blue;
                Console.WriteLine($"Importing {i + 1} of {filesToIngest.Length}: {path}");
                await memory.ImportDocumentAsync(path, steps: Constants.PipelineWithoutSummary);
                Console.WriteLine($"Completed in {sw.Elapsed}\n");
            }

            // Ask a predefined question
            Console.ForegroundColor = ConsoleColor.Green;
            string question1 = "What formats does KM support";
            Console.WriteLine($"Question: {question1}");
            await AnswerQuestion(memory, question1);

            // Let the user ask additional questions
            while (true)
            {
                Console.ForegroundColor = ConsoleColor.Green;
                Console.Write("Question: ");
                string question = Console.ReadLine()!;
                if (string.IsNullOrEmpty(question))
                    return;

                await AnswerQuestion(memory, question);
            }
        }

        private static IKernelMemory CreateMemory(string modelPath)
        {
            Common.InferenceParams infParams = new() { AntiPrompts = ["\n\n"] };

            LLamaSharpConfig lsConfig = new(modelPath) { DefaultInferenceParams = infParams };

            SearchClientConfig searchClientConfig = new() 
            {
                MaxMatchesCount = 1,
                AnswerTokens = 100,
            };

            TextPartitioningOptions parseOptions = new()
            {
                MaxTokensPerParagraph = 300,
                MaxTokensPerLine = 100,
                OverlappingTokens = 30
            };

            return new KernelMemoryBuilder()
                .WithLLamaSharpDefaults(lsConfig)
                .WithSearchClientConfig(searchClientConfig)
                .With(parseOptions)
                .Build();
        }

        private static async Task AnswerQuestion(IKernelMemory memory, string question)
        {
            Stopwatch sw = Stopwatch.StartNew();
            Console.ForegroundColor = ConsoleColor.DarkGray;
            Console.WriteLine($"Generating answer...");

            MemoryAnswer answer = await memory.AskAsync(question);
            Console.WriteLine($"Answer generated in {sw.Elapsed}");

            Console.ForegroundColor = ConsoleColor.Gray;
            Console.WriteLine($"Answer: {answer.Result}");
            foreach (var source in answer.RelevantSources)
            {
                Console.WriteLine($"Source: {source.SourceName}");
            }
            Console.WriteLine();
        }
    }
}
```

--------------------------------

### Message Property

Source: https://scisharp.github.io/LLamaSharp/0.25.0/xmldocs/llama.batched.cannotsavewhilerequiresinferenceexception

Gets a message that describes the current exception. This is a standard property for all exceptions.

```csharp
public string Message { get; }
```