From 4a7a9c80282f603a8600438b9a1589943180bfa0 Mon Sep 17 00:00:00 2001 From: Lily Du Date: Wed, 9 Oct 2024 13:42:05 -0700 Subject: [PATCH] [repo] chore: add streaming to CONCEPTS docs (#2100) ## Linked issues closes: #1969 ## Attestation Checklist - [x] My code follows the style guidelines of this project - I have checked for/fixed spelling, linting, and other errors - I have commented my code for clarity - I have made corresponding changes to the documentation (updating the doc strings in the code is sufficient) - My changes generate no new warnings - I have added tests that validates my changes, and provides sufficient test coverage. I have tested with: - Local testing - E2E testing in Teams - New and existing unit tests pass locally with my changes --- getting-started/CONCEPTS/ACTION-PLANNER.md | 1 + getting-started/CONCEPTS/ACTIONS.md | 1 + getting-started/CONCEPTS/AI-SYSTEM.md | 1 + getting-started/CONCEPTS/APPLICATION.md | 1 + .../CONCEPTS/ASSISTANTS-PLANNER.md | 1 + getting-started/CONCEPTS/AUGMENTATIONS.md | 1 + getting-started/CONCEPTS/DATA-SOURCES.md | 1 + getting-started/CONCEPTS/FUNCTION-CALLS.md | 1 + getting-started/CONCEPTS/MODERATOR.md | 1 + getting-started/CONCEPTS/PLANNER.md | 1 + getting-started/CONCEPTS/POWERED-BY-AI.md | 1 + getting-started/CONCEPTS/PROMPTS.md | 1 + getting-started/CONCEPTS/README.md | 1 + getting-started/CONCEPTS/STREAMING.md | 165 ++++++++++++++++++ getting-started/CONCEPTS/TURNS.md | 1 + getting-started/CONCEPTS/USER-AUTH.md | 1 + 16 files changed, 180 insertions(+) create mode 100644 getting-started/CONCEPTS/STREAMING.md diff --git a/getting-started/CONCEPTS/ACTION-PLANNER.md b/getting-started/CONCEPTS/ACTION-PLANNER.md index 701ee62b0..a34748b3a 100644 --- a/getting-started/CONCEPTS/ACTION-PLANNER.md +++ b/getting-started/CONCEPTS/ACTION-PLANNER.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/ACTIONS.md b/getting-started/CONCEPTS/ACTIONS.md index 0242e98c0..7184384dd 100644 --- a/getting-started/CONCEPTS/ACTIONS.md +++ b/getting-started/CONCEPTS/ACTIONS.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/AI-SYSTEM.md b/getting-started/CONCEPTS/AI-SYSTEM.md index b6cf023d3..bd9f2df82 100644 --- a/getting-started/CONCEPTS/AI-SYSTEM.md +++ b/getting-started/CONCEPTS/AI-SYSTEM.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/APPLICATION.md b/getting-started/CONCEPTS/APPLICATION.md index 8f08328ec..9a9454010 100644 --- a/getting-started/CONCEPTS/APPLICATION.md +++ b/getting-started/CONCEPTS/APPLICATION.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/ASSISTANTS-PLANNER.md b/getting-started/CONCEPTS/ASSISTANTS-PLANNER.md index 66c073704..6fc3a18b2 100644 --- a/getting-started/CONCEPTS/ASSISTANTS-PLANNER.md +++ b/getting-started/CONCEPTS/ASSISTANTS-PLANNER.md @@ -13,6 +13,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) - [**Assistants Planner**](./ASSISTANTS-PLANNER.md) diff --git a/getting-started/CONCEPTS/AUGMENTATIONS.md b/getting-started/CONCEPTS/AUGMENTATIONS.md index 5970717c8..73ce07224 100644 --- a/getting-started/CONCEPTS/AUGMENTATIONS.md +++ b/getting-started/CONCEPTS/AUGMENTATIONS.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/DATA-SOURCES.md b/getting-started/CONCEPTS/DATA-SOURCES.md index 25ae93645..64b2fdbd8 100644 --- a/getting-started/CONCEPTS/DATA-SOURCES.md +++ b/getting-started/CONCEPTS/DATA-SOURCES.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/FUNCTION-CALLS.md b/getting-started/CONCEPTS/FUNCTION-CALLS.md index cc7bc92c3..c9ac93c9d 100644 --- a/getting-started/CONCEPTS/FUNCTION-CALLS.md +++ b/getting-started/CONCEPTS/FUNCTION-CALLS.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/MODERATOR.md b/getting-started/CONCEPTS/MODERATOR.md index 096820bcb..933e9bc83 100644 --- a/getting-started/CONCEPTS/MODERATOR.md +++ b/getting-started/CONCEPTS/MODERATOR.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/PLANNER.md b/getting-started/CONCEPTS/PLANNER.md index 502defbda..0b1b6e09c 100644 --- a/getting-started/CONCEPTS/PLANNER.md +++ b/getting-started/CONCEPTS/PLANNER.md @@ -14,6 +14,7 @@ - [**Planner**](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/POWERED-BY-AI.md b/getting-started/CONCEPTS/POWERED-BY-AI.md index 499503195..41595ba0c 100644 --- a/getting-started/CONCEPTS/POWERED-BY-AI.md +++ b/getting-started/CONCEPTS/POWERED-BY-AI.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [**Powered by AI**](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/PROMPTS.md b/getting-started/CONCEPTS/PROMPTS.md index 76bd67719..97175e684 100644 --- a/getting-started/CONCEPTS/PROMPTS.md +++ b/getting-started/CONCEPTS/PROMPTS.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [**Prompts**](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/README.md b/getting-started/CONCEPTS/README.md index 9cd58bbbd..cb54d5152 100644 --- a/getting-started/CONCEPTS/README.md +++ b/getting-started/CONCEPTS/README.md @@ -13,6 +13,7 @@ Here you will find short guides on features available in the library. | [Data Sources](DATA-SOURCES.md) | Describes the usage of data sources in the library. | | [User Authentication](USER-AUTH.md) | Describes user authentication features out of the box | | [Powered by AI](./POWERED-BY-AI.md) | Describes the Bot UX features available in this SDK. | +| [Streaming](./STREAMING.md) | Describes the Bot Streaming feature. | ## The AI System Concepts diff --git a/getting-started/CONCEPTS/STREAMING.md b/getting-started/CONCEPTS/STREAMING.md new file mode 100644 index 000000000..c99ff131e --- /dev/null +++ b/getting-started/CONCEPTS/STREAMING.md @@ -0,0 +1,165 @@ +# Streaming for Bots +***NOTE: This feature is in the rollout phase and is available only to specific tenants. Our team is actively working on enabling this feature fully on Teams and across all languages in the SDK. Rest assured; we are diligently working to enable this feature for everyone. Updates will be posted on the Discussions page.*** + +**Navigation** + +- [00.OVERVIEW](./README.md) +- [Action Planner](./ACTION-PLANNER.md) +- [Actions](./ACTIONS.md) +- [AI System](./AI-SYSTEM.md) +- [Application class](./APPLICATION.md) +- [Augmentations](./AUGMENTATIONS.md) +- [Data Sources](./DATA-SOURCES.md) +- [Function Calls](./FUNCTION-CALLS.md) +- [Moderator](./MODERATOR.md) +- [Planner](./PLANNER.md) +- [Powered by AI](./POWERED-BY-AI.md) +- [Prompts](./PROMPTS.md) +- [**Streaming**](./STREAMING.md) +- [Turns](./TURNS.md) +- [User Authentication](./USER-AUTH.md) + +--- + +AI-powered bots tend to have slower response times which can disengage users. There are two factors that contribute to a slow response. The first is the multiple preprocessing steps such as RAG or function calls which take time and are often required before the LLM can produce a response. The second is the time the LLM takes to generate a full response. + +A common solution is to stream the bot’s response to users while the LLM generates its full response. Through streaming, your bot can offer an experience that feels engaging, responsive, and on-par with leading AI products. + +There are two parts to streaming: + +- **Informative Updates**: Provide users with insights into what your bot is doing before it has started generating its response. + +- **Response Streaming**: Provide users with chunks of the response as they are generated by the LLM. This feels like the bot is actively typing out its message. + +## Sample Bots +- [C# Streaming ChefBot](https://github.com/microsoft/teams-ai/tree/main/dotnet/samples/04.ai.g.teamsChefBot-streaming) +- [JS Streaming ChefBot](https://github.com/microsoft/teams-ai/tree/main/js/samples/04.ai-apps/i.teamsChefBot-streaming) + +## Streaming Response Class +The `StreamingResponse` class is the helper class for streaming responses to the client. The class is used to send a series of updates to the client in a single response. If you are using your own custom model, you can directly instantiate and manage this class to stream responses. + +The expected sequence of calls is: + +1. `queueInformativeUpdate()` +2. `queueTextChunk()`, ..., +3. `endStream()`. + +Once `endStream()` is called, the stream is considered ended and no further updates can be sent. + + +## Configuration with Azure Open AI / Open AI + +### Current Limitations: +- Streaming is only available in 1:1 chats. +- Only rich text can be streamed. +- Only one informative message can be set. This is reused for each message. + - Examples include: + - “Scanning through documents” + - “Summarizing content” + - “Finding relevant work items” +- The informative message is rendered only at the beginning of each message returned from the LLM. +- Attachments can only be sent in the final streamed chunk. +- Streaming is not available in conjunction with AI SDK's function calls yet. + + +### Setup Instructions: +You can configure streaming with your bot by following these steps: + +- Use the `DefaultAugmentation` class +- Set `stream: true` in the `OpenAIModel` declaration + + +#### Optional additions: +- Set the informative message in the `ActionPlanner` declaration via the `StartStreamingMessage` config. +- Set attachments in the final chunk via the `EndStreamHandler` in the `ActionPlanner` declaration. + +#### C# + +```cs + // Create OpenAI Model + builder.Services.AddSingleton (sp => new( + new OpenAIModelOptions(config.OpenAI.ApiKey, "gpt-4o") + { + LogRequests = true, + Stream = true, // Set stream toggle + }, + sp.GetService() + )); + +ResponseReceivedHandler endStreamHandler = new((object sender, ResponseReceivedEventArgs args) => + { + StreamingResponse? streamer = args.Streamer; + + if (streamer == null) + { + return; + } + + AdaptiveCard adaptiveCard = new("1.6") + { + Body = [new AdaptiveTextBlock(streamer.Message) { Wrap = true }] + }; + + var adaptiveCardAttachment = new Attachment() + { + ContentType = "application/vnd.microsoft.card.adaptive", + Content = adaptiveCard, + }; + + + streamer.Attachments = [adaptiveCardAttachment]; // Set attachments + + }); + + + // Create ActionPlanner + ActionPlanner planner = new( + options: new( + model: sp.GetService()!, + prompts: prompts, + defaultPrompt: async (context, state, planner) => + { + PromptTemplate template = prompts.GetPrompt("Chat"); + return await Task.FromResult(template); + } + ) + { + LogRepairs = true, + StartStreamingMessage = "Loading stream results...", // Set informative message + EndStreamHandler = endStreamHandler // Set final chunk handler + }, + loggerFactory: loggerFactory + ); +``` + +#### JS/TS + +```js +const model = new OpenAIModel({ + // ...Setup OpenAI or AzureOpenAI + stream: true, // Set stream toggle +}); + +const endStreamHandler: PromptCompletionModelResponseReceivedEvent = (ctx, memory, response, streamer) => { + // ... Setup attachments + streamer.setAttachments([...cards]); // Set attachments +}; + +const planner = new ActionPlanner({ + model, + prompts, + defaultPrompt: 'default', + startStreamingMessage: 'Loading stream results...', // Set informative message + endStreamHandler: endStreamHandler // Set final chunk handler +}); +``` + +--- + +## Return to other major section topics: + +- [**CONCEPTS**](../CONCEPTS/README.md) +- [MIGRATION](../MIGRATION/README.md) +- [QUICKSTART](../QUICKSTART.md) +- [SAMPLES](../SAMPLES.md) +- [OTHER](../OTHER/README.md) diff --git a/getting-started/CONCEPTS/TURNS.md b/getting-started/CONCEPTS/TURNS.md index 765fd3a81..bc2653cf6 100644 --- a/getting-started/CONCEPTS/TURNS.md +++ b/getting-started/CONCEPTS/TURNS.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [**Turns**](./TURNS.md) - [User Authentication](./USER-AUTH.md) diff --git a/getting-started/CONCEPTS/USER-AUTH.md b/getting-started/CONCEPTS/USER-AUTH.md index f09a67ea1..9e9f8d9fe 100644 --- a/getting-started/CONCEPTS/USER-AUTH.md +++ b/getting-started/CONCEPTS/USER-AUTH.md @@ -14,6 +14,7 @@ - [Planner](./PLANNER.md) - [Powered by AI](./POWERED-BY-AI.md) - [Prompts](./PROMPTS.md) +- [Streaming](./STREAMING.md) - [Turns](./TURNS.md) - [**User Authentication**](./USER-AUTH.md)