#!/usr/bin/env python # coding: utf-8 # # Streaming Results # # Here is an example pattern if you want to stream your multiple results. Note that this is not supported for Hugging Face text completions at this time. # # Import Semantic Kernel SDK from pypi.org # In[ ]: # Note: if using a virtual environment, do not run this cell get_ipython().run_line_magic('pip', 'install -U semantic-kernel') from semantic_kernel import __version__ __version__ # Initial configuration for the notebook to run properly. # In[ ]: # Make sure paths are correct for the imports import os import sys notebook_dir = os.path.abspath("") parent_dir = os.path.dirname(notebook_dir) grandparent_dir = os.path.dirname(parent_dir) sys.path.append(grandparent_dir) # ### Configuring the Kernel # # Let's get started with the necessary configuration to run Semantic Kernel. For Notebooks, we require a `.env` file with the proper settings for the model you use. Create a new file named `.env` and place it in this directory. Copy the contents of the `.env.example` file from this directory and paste it into the `.env` file that you just created. # # **NOTE: Please make sure to include `GLOBAL_LLM_SERVICE` set to either OpenAI, AzureOpenAI, or HuggingFace in your .env file. If this setting is not included, the Service will default to AzureOpenAI.** # # #### Option 1: using OpenAI # # Add your [OpenAI Key](https://openai.com/product/) key to your `.env` file (org Id only if you have multiple orgs): # # ``` # GLOBAL_LLM_SERVICE="OpenAI" # OPENAI_API_KEY="sk-..." # OPENAI_ORG_ID="" # OPENAI_CHAT_MODEL_ID="" # OPENAI_TEXT_MODEL_ID="" # OPENAI_EMBEDDING_MODEL_ID="" # ``` # The names should match the names used in the `.env` file, as shown above. # # #### Option 2: using Azure OpenAI # # Add your [Azure Open AI Service key](https://learn.microsoft.com/azure/cognitive-services/openai/quickstart?pivots=programming-language-studio) settings to the `.env` file in the same folder: # # ``` # GLOBAL_LLM_SERVICE="AzureOpenAI" # AZURE_OPENAI_API_KEY="..." # AZURE_OPENAI_ENDPOINT="https://..." # AZURE_OPENAI_CHAT_DEPLOYMENT_NAME="..." # AZURE_OPENAI_TEXT_DEPLOYMENT_NAME="..." # AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME="..." # AZURE_OPENAI_API_VERSION="..." # ``` # The names should match the names used in the `.env` file, as shown above. # # For more advanced configuration, please follow the steps outlined in the [setup guide](./CONFIGURING_THE_KERNEL.md). # We will load our settings and get the LLM service to use for the notebook. # In[ ]: from services import Service from samples.service_settings import ServiceSettings service_settings = ServiceSettings.create() # Select a service to use for this notebook (available services: OpenAI, AzureOpenAI, HuggingFace) selectedService = ( Service.AzureOpenAI if service_settings.global_llm_service is None else Service(service_settings.global_llm_service.lower()) ) print(f"Using service type: {selectedService}") # First, we will set up the text and chat services we will be submitting prompts to. # # In[ ]: from semantic_kernel import Kernel from semantic_kernel.connectors.ai.open_ai import ( AzureChatCompletion, AzureChatPromptExecutionSettings, # noqa: F401 AzureTextCompletion, OpenAIChatCompletion, OpenAIChatPromptExecutionSettings, # noqa: F401 OpenAITextCompletion, OpenAITextPromptExecutionSettings, # noqa: F401 ) from semantic_kernel.contents import ChatHistory # noqa: F401 kernel = Kernel() service_id = None if selectedService == Service.OpenAI: from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion service_id = "default" oai_chat_service = OpenAIChatCompletion( service_id="oai_chat", ) oai_text_service = OpenAITextCompletion( service_id="oai_text", ) elif selectedService == Service.AzureOpenAI: from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion service_id = "default" aoai_chat_service = AzureChatCompletion( service_id="aoai_chat", ) aoai_text_service = AzureTextCompletion( service_id="aoai_text", ) # Configure Hugging Face service if selectedService == Service.HuggingFace: from semantic_kernel.connectors.ai.hugging_face import ( HuggingFacePromptExecutionSettings, # noqa: F401 HuggingFaceTextCompletion, ) hf_text_service = HuggingFaceTextCompletion(ai_model_id="distilgpt2", task="text-generation") # Next, we'll set up the completion request settings for text completion services. # # In[ ]: oai_prompt_execution_settings = OpenAITextPromptExecutionSettings( service_id="oai_text", max_tokens=150, temperature=0.7, top_p=1, frequency_penalty=0.5, presence_penalty=0.5, ) # ## Streaming Open AI Text Completion # # In[ ]: if selectedService == Service.OpenAI: prompt = "What is the purpose of a rubber duck?" stream = oai_text_service.get_streaming_text_contents(prompt=prompt, settings=oai_prompt_execution_settings) async for message in stream: print(str(message[0]), end="") # end = "" to avoid newlines # ## Streaming Azure Open AI Text Completion # # In[ ]: if selectedService == Service.AzureOpenAI: prompt = "provide me a list of possible meanings for the acronym 'ORLD'" stream = aoai_text_service.get_streaming_text_contents(prompt=prompt, settings=oai_prompt_execution_settings) async for message in stream: print(str(message[0]), end="") # ## Streaming Hugging Face Text Completion # # In[ ]: if selectedService == Service.HuggingFace: hf_prompt_execution_settings = HuggingFacePromptExecutionSettings( service_id="hf_text", extension_data={ "max_new_tokens": 80, "top_p": 1, "eos_token_id": 11, "pad_token_id": 0, }, ) # In[ ]: if selectedService == Service.HuggingFace: prompt = "The purpose of a rubber duck is" stream = hf_text_service.get_streaming_text_contents( prompt=prompt, prompt_execution_settings=hf_prompt_execution_settings ) async for text in stream: print(str(text[0]), end="") # end = "" to avoid newlines # Here, we're setting up the settings for Chat completions. # # In[ ]: oai_chat_prompt_execution_settings = OpenAIChatPromptExecutionSettings( service_id="oai_chat", max_tokens=150, temperature=0.7, top_p=1, frequency_penalty=0.5, presence_penalty=0.5, ) # ## Streaming OpenAI Chat Completion # # In[ ]: if selectedService == Service.OpenAI: content = "You are an AI assistant that helps people find information." chat = ChatHistory() chat.add_system_message(content) stream = oai_chat_service.get_streaming_chat_message_contents( chat_history=chat, settings=oai_chat_prompt_execution_settings ) async for text in stream: print(str(text[0]), end="") # end = "" to avoid newlines # ## Streaming Azure OpenAI Chat Completion # # In[ ]: az_oai_chat_prompt_execution_settings = AzureChatPromptExecutionSettings( service_id="aoai_chat", max_tokens=150, temperature=0.7, top_p=1, frequency_penalty=0.5, presence_penalty=0.5, ) # In[ ]: if selectedService == Service.AzureOpenAI: content = "You are an AI assistant that helps people find information." chat = ChatHistory() chat.add_system_message(content) chat.add_user_message("What is the purpose of a rubber duck?") stream = aoai_chat_service.get_streaming_chat_message_contents( chat_history=chat, settings=az_oai_chat_prompt_execution_settings ) async for text in stream: print(str(text[0]), end="") # end = "" to avoid newlines