#r "nuget:Microsoft.DotNet.Interactive.AI, 1.0.0-beta.23604.2"
#!value --name key
YOUR AZURE OPEN AI KEY
#!value --name endpoint
https://your-enpoint.openai.azure.com/
Note: If your deployment names are different, you will need to change --deployment
from text-embedding-ada-002
and gpt-35-turbo-16k
in the lines below:
#!connect azure-openai --model-type TextEmbeddingGenerator --kernel-name knowledge --api-key @value:key --endpoint @value:endpoint --deployment text-embedding-ada-002
#!connect azure-openai --model-type ChatCompletion --kernel-name chat --api-key @value:key --endpoint @value:endpoint --deployment gpt-35-turbo-16k --use-knowledge knowledge
#r "nuget: HtmlAgilityPack, 1.11.49"
The SetTransformation
method is then called to set a transformation function that takes in text and a MIME type and returns a list of perspectives. The transformation function first checks if the MIME type is "text/html" and if so, extracts the plain text from the HTML using the HtmlAgilityPack
library.
The function then uses the Markdig
library to parse the input text into a Markdown document. The function filters the paragraphs from the Markdown document and generates a summary and overview for each paragraph using the textKernel
. The take away points, summary, and overview are added to a list of perspectives.
If the length of the paragraph is greater than 1000 characters, the entire paragraph is added to the list of perspectives. The function returns the list of perspectives.
This code is useful for generating perspectives on input text using machine learning models. The knowledgeKernel
will then use the data generated to populate the semantic memory and use text embeddings to retrieve relevant data to focus questions sent to the TextCompletionKernel
.
mermaid
sequenceDiagram
participant User
participant KnowledgeKernel
participant TextCompletionKernel
User->>KnowledgeKernel: Sends text to be stored in the KnowledgeKernel
KnowledgeKernel->>KnowledgeKernel: Extracts plain text from input
KnowledgeKernel->>KnowledgeKernel: Parses input using MarkdownPipelineBuilder
KnowledgeKernel->>KnowledgeKernel: Filters paragraphs from the parsed input
loop Processing each paragraph
KnowledgeKernel->>TextCompletionKernel: Sends a prompt to summarize the paragraph
TextCompletionKernel->>KnowledgeKernel: Extracts the take away points from the summary
KnowledgeKernel->>TextCompletionKernel: Sends a prompt to summarize the paragraph in 5 lines
TextCompletionKernel->>KnowledgeKernel: Extracts the summary from the prompt
KnowledgeKernel->>TextCompletionKernel: Sends a prompt to provide the most important information
TextCompletionKernel->>KnowledgeKernel: Extracts the overview from the prompt
end
KnowledgeKernel->>User: Returns the take away points, summary, and overview
using Microsoft.DotNet.Interactive.AI;
using Microsoft.DotNet.Interactive;
using Microsoft.DotNet.Interactive.Formatting;
using Microsoft.DotNet.Interactive.Commands;
using Microsoft.DotNet.Interactive.Events;
using HtmlAgilityPack;
using Markdig;
using Markdig.Helpers;
using Markdig.Parsers;
using Markdig.Parsers.Inlines;
using Markdig.Syntax;
List<(string text, Dictionary<string,object>? metadata)> lastData = null;
var knowledgeKernel = Kernel.Root.FindKernelByName("knowledge") as KnowledgeKernel;
var textKernel = Kernel.Root.FindKernelByName("chat(text)") as TextCompletionKernel;
knowledgeKernel.SetTransformation(async (text,mimeType)=>{
textKernel.SetSuppressDisplay(true);
var input = text;
if(mimeType == "text/html"){
var doc = new HtmlDocument();
doc.LoadHtml(text);
input = doc.DocumentNode.InnerText;
}
var pipeline = new MarkdownPipelineBuilder().UseAdvancedExtensions().Build();
var document = Markdown.Parse(input, pipeline);
var paragraphs = document.Where(d => d is ParagraphBlock);
var perspectives = new List<(string text, Dictionary<string,object>? metadata)>();
foreach (var paragraph in paragraphs)
{
var block = input.Substring( paragraph.Span.Start, paragraph.Span.Length);
var result = await textKernel.SendAsync(new SubmitCode(
$"""
summararize the following text providing the take away points:
{block}
"""));
var takeAway = result.Events.OfType<ReturnValueProduced>().LastOrDefault()?.FormattedValues.First(f => f.MimeType == PlainTextFormatter.MimeType).Value.Trim();
result = await textKernel.SendAsync(new SubmitCode(
$"""
summararize the following text with up to 5 lines
{block}
"""));
var summary = result.Events.OfType<ReturnValueProduced>().LastOrDefault()?.FormattedValues.First(f => f.MimeType == PlainTextFormatter.MimeType).Value.Trim();
result = await textKernel.SendAsync(new SubmitCode(
$"""
given
{takeAway}
and
{block}
what is the most important information to know?
"""));
var overView = result.Events.OfType<ReturnValueProduced>().LastOrDefault()?.FormattedValues.First(f => f.MimeType == PlainTextFormatter.MimeType).Value.Trim();
perspectives.Add((takeAway, null));
perspectives.Add((summary, null));
perspectives.Add((overView, null));
// comment the following lines to avoid printing all results
$"""
Storing derived knowledge:
takeAway
---
{takeAway}
---
summary---
{summary}
---
overView---
{overView}
---
""".Display();
if (block.Length > 1000)
{
perspectives.Add((block, null));
}
}
lastData = perspectives;
textKernel.SetSuppressDisplay(false);
return perspectives;
});
You can skip this step if you just want to load persisted vector store
#!knowledge --name fact1
.NET Interactive is a polyglot kernel.
#!knowledge --name fact2
.NET Interactive is developed by a team of engineers at Microsoft.
#!knowledge --name fact3
.NET Interactive is an open source project with a lot of contributors.
#!knowledge --name fact4
.NET Interactive is used by engineers.
#!knowledge --name fact5
.NET Interactive is developed by open source community.
#!knowledge --name fact6
.NET Interactive is used by software engineers.
#!knowledge --name fact7
.NET Interactive can create rich outputs and visualizations.
#!knowledge --name fact8
The Polyglot Notebooks extension, powered by .NET Interactive, brings support for multi-language notebooks to Visual Studio Code.
Classic notebook software typically supports notebooks that use only one language at a time. With Polyglot Notebooks, features such as completions, documentation, syntax highlighting, and diagnostics are available for many languages in one notebook. In addition, different cells in the same notebook can run in separate processes or on different machines, allowing a notebook to span local and cloud environments in one combined workflow.
Polyglot Notebooks are fully interoperable with Jupyter and support the .ipynb file extension.
You don't need to choose between the capabilities of Polyglot Notebooks and the rich Jupyter ecosystem.
If your notebook is saved in the .ipynb format, you can open it in Jupyter and the cell languages will still be recognized.
When working in Jupyter using the .NET Interactive kernel, you can switch cell languages using magic commands.
#!knowledge --name fact9
.NET Interactive has support for mermaid langauge and javascript. Along with the HTML kernel these components can be used to create rich visualizations. With the mermaid language you can create diagrams and flowcharts. With javascript you can create interactive visualizations. With HTML you can create rich visualizations.
#!knowledge --name fact10
.NET Interactive alloes using languages like Python, R and Julia thanks to the Jupyter protocol support. This allows you to use the rich ecosystem of these languages in your .NET Interactive notebooks.
#!knowledge --name fact20 --from-url https://raw.githubusercontent.com/dotnet/interactive/main/docs/FAQ.md
SHOW TABLES;
select id, payload from defaultVectorCollection limit 5;
using Microsoft.DotNet.Interactive.AI.DuckDb;
using System.IO;
var currentDirectory = Directory.GetCurrentDirectory();
var dataFile = Path.Combine(currentDirectory, "interactive_knowledge.json");
knowledge kernel
¶using Microsoft.DotNet.Interactive.AI.DuckDb;
await Kernel.Root.SendAsync(new ExportCollection("defaultVectorCollection", dataFile, "knowledge(duckDbVector)"));
using Microsoft.DotNet.Interactive.AI.DuckDb;
await Kernel.Root.SendAsync(new ImportCollection("defaultVectorCollection", dataFile, "knowledge(duckDbVector)"));
what can i do with .NET Interactive?
does it support python?
#!function be_clear
Answer the following questions:
{{$input}}
And make sure to be succint and clear.
#!use-skills function._GLOBAL_FUNCTIONS_.be_clear
Why can I do data visualizations with .NET Interactive?
What should I do if a cell runs forever?
Can I create custom kernels for langauges and tools that are not supported by the Jupyter Notebook?