🧬 Embedding Strategies in LM-Kit: Rapid Prototyping to Durable Storage

				
					    // Define some strings from which we want to generate embeddings
    string[] examples =
    {
        "How do I bake a chocolate cake?",
        "What is the recipe for chocolate cake?",
        "I want to make a chocolate cake.",
        "Chocolate cake is delicious.",
        "How do I cook pasta?",
        "I need instructions to bake a cake.",
        "Baking requires precise measurements.",
        "I like vanilla ice cream.",
        "The weather is sunny today.",
        "What is the capital of France?",
        "Paris is a beautiful city.",
        "How can I improve my coding skills?",
        "Programming requires practice."
    };

    // Load the embedding model
    var model = LM.LoadFromModelID("nomic-embed-text");

    // Specify optional metadata to attach to the new collection (a.k.a. DataSource)
    var collectionMetadata = new MetadataCollection
    {
      { "description", "my description" },
      { "another-pair-key", "another-pair-value" }
    };

    // Create a new in-memory vector database

    var collection = DataSource.CreateInMemoryDataSource(
        "my-collection",
        model,
        collectionMetadata);

    // Compute embeddings to insert into the collection
    var embedder = new Embedder(model);
    List<DataSource.VectorEntry> vectorEntries = new List<DataSource.VectorEntry>();

    // Run multithreaded embedding on the list of examples
    var embeddings = embedder.GetEmbeddings(examples);

    for (int index = 0; index < examples.Length; index++)
    {
        vectorEntries.Add(new DataSource.VectorEntry(
            vector: embeddings[index],
            payload: examples[index]));
    }

    const string SectionIdentifier = "my-section-identifier";

    // Specify optional metadata to attach to the new section.
    // Note: a single collection can contain multiple sections.
    var sectionMetadata = new MetadataCollection
    {
      { "description", "my description" },
      { "another-pair-key", "another-pair-value" }
    };

    // Add the computed embedding vectors to the collection
    collection.Upsert(
        SectionIdentifier,
        vectorEntries,
        sectionMetadata);

    //now perform search

    // Build the query vector
    string query = "How do I bake a chocolate cake?";
    var queryVector = new Embedder(model).GetEmbeddings(query);

    // Search for similar vectors across partitions
    var similarPartitions = VectorSearch.FindMatchingPartitions(
        [collection],
        model,
        queryVector);
    // Do something with the search results...