From 0de357dabbe2dc4279a875b21c9d9e7f0a5fdd10 Mon Sep 17 00:00:00 2001 From: Devis Lucato Date: Tue, 12 Nov 2024 16:52:10 -0800 Subject: [PATCH] * Upgrade Semantic Kernel * Upgrade Kernel Memory * Fix Github plugin * Handle OpenAI plugins deprecation * Cleanup code, refactoring, code style --- .gitignore | 2 + CopilotChat.sln.DotSettings | 1 + Directory.Build.props | 51 ++++ Directory.Packages.props | 102 +++++++ .../ChatCopilotIntegrationTest.cs | 7 +- .../ChatCopilotIntegrationTests.csproj | 47 ++-- integration-tests/ChatTests.cs | 10 +- integration-tests/HealthzTests.cs | 1 - integration-tests/ServiceInfoTests.cs | 7 +- integration-tests/SpeechTokenTests.cs | 5 +- integration-tests/StaticFiles.cs | 1 - .../CopilotChatMemoryPipeline.csproj | 25 +- memorypipeline/Program.cs | 6 - plugins/shared/PluginShared.csproj | 20 -- plugins/web-searcher/PluginEndpoint.cs | 25 +- plugins/web-searcher/WebSearcher.csproj | 30 +- shared/ConfigurationBuilderExtensions.cs | 2 - shared/CopilotChatShared.csproj | 37 +-- shared/MemoryClientBuilderExtensions.cs | 25 -- shared/Ocr/ConfigurationExtensions.cs | 41 --- ...TesseractOptions.cs => TesseractConfig.cs} | 4 +- shared/Ocr/Tesseract/TesseractOcrEngine.cs | 22 +- shared/ServiceConfiguration.cs | 118 +++++--- tools/importdocument/ImportDocument.csproj | 29 +- tools/importdocument/Program.cs | 7 - webapi/Auth/AuthInfo.cs | 2 - .../ChatParticipantAuthorizationHandler.cs | 6 +- .../Auth/PassThroughAuthenticationHandler.cs | 2 - webapi/Controllers/ChatArchiveController.cs | 7 - webapi/Controllers/ChatController.cs | 73 +++-- webapi/Controllers/ChatHistoryController.cs | 7 - webapi/Controllers/ChatMemoryController.cs | 6 - .../Controllers/ChatParticipantController.cs | 4 - webapi/Controllers/DocumentController.cs | 7 - webapi/Controllers/MaintenanceController.cs | 3 - webapi/Controllers/PluginController.cs | 6 - webapi/Controllers/ServiceInfoController.cs | 6 - webapi/Controllers/SpeechTokenController.cs | 4 - webapi/CopilotChatWebApi.csproj | 70 ++--- .../Extensions/AsyncEnumerableExtensions.cs | 3 - webapi/Extensions/ConfigurationExtensions.cs | 3 - webapi/Extensions/ExceptionExtensions.cs | 2 - .../KernelMemoryClientExtensions.cs | 34 ++- webapi/Extensions/SemanticKernelExtensions.cs | 9 - webapi/Extensions/ServiceExtensions.cs | 7 - webapi/Hubs/MessageRelayHub.cs | 2 - webapi/Models/Request/Ask.cs | 2 - webapi/Models/Request/DocumentImportForm.cs | 4 - webapi/Models/Request/DocumentStatusForm.cs | 4 - webapi/Models/Response/AskResult.cs | 3 - webapi/Models/Response/ChatArchive.cs | 1 - .../Models/Response/DocumentMessageContent.cs | 2 - webapi/Models/Response/ServiceInfoResponse.cs | 2 - webapi/Models/Storage/ChatParticipant.cs | 1 - webapi/Models/Storage/ChatSession.cs | 2 - webapi/Models/Storage/CopilotChatMessage.cs | 2 - webapi/Models/Storage/MemorySource.cs | 1 - webapi/Options/DocumentMemoryOptions.cs | 1 - webapi/Options/MemoryStoreType.cs | 2 - .../Options/NotEmptyOrWhitespaceAttribute.cs | 1 - webapi/Options/PluginOptions.cs | 2 - webapi/Options/PromptsOptions.cs | 2 - .../RequiredOnPropertyValueAttribute.cs | 1 - webapi/Plugins/Chat/ChatPlugin.cs | 6 - webapi/Plugins/Chat/KernelMemoryRetriever.cs | 31 +-- webapi/Plugins/Chat/MsGraphOboPlugin.cs | 9 +- webapi/Plugins/Chat/SemanticChatMemory.cs | 2 - .../Chat/SemanticChatMemoryExtractor.cs | 19 +- .../OpenApi/GitHubPlugin/Model/PullRequest.cs | 6 +- .../Plugins/OpenApi/GitHubPlugin/openapi.json | 6 +- .../JiraPlugin/Model/CommentResponse.cs | 1 - webapi/Plugins/Utils/AsyncUtils.cs | 2 - webapi/Plugins/Utils/JsonUtils.cs | 4 +- webapi/Plugins/Utils/TokenUtils.cs | 6 +- webapi/Program.cs | 8 - .../Services/AppInsightsTelemetryService.cs | 2 - ...InsightsUserTelemetryInitializerService.cs | 1 - webapi/Services/AzureContentSafety.cs | 7 - webapi/Services/DocumentTypeProvider.cs | 2 - webapi/Services/IContentSafetyService.cs | 5 - webapi/Services/IMaintenanceAction.cs | 3 - webapi/Services/MaintenanceMiddleware.cs | 4 - webapi/Services/SemanticKernelProvider.cs | 4 - webapi/Storage/ChatMemorySourceRepository.cs | 3 - webapi/Storage/ChatMessageRepository.cs | 3 - webapi/Storage/ChatParticipantRepository.cs | 3 - webapi/Storage/ChatSessionRepository.cs | 2 - webapi/Storage/CosmosDbContext.cs | 4 - webapi/Storage/FileSystemContext.cs | 5 - webapi/Storage/IRepository.cs | 3 - webapi/Storage/IStorageContext.cs | 3 - webapi/Storage/Repository.cs | 3 - webapi/Storage/VolatileContext.cs | 4 - webapi/Utilities/PluginUtils.cs | 11 +- webapi/appsettings.json | 261 +++++++++++++----- 95 files changed, 618 insertions(+), 736 deletions(-) create mode 100644 Directory.Build.props create mode 100644 Directory.Packages.props delete mode 100644 shared/MemoryClientBuilderExtensions.cs delete mode 100644 shared/Ocr/ConfigurationExtensions.cs rename shared/Ocr/Tesseract/{TesseractOptions.cs => TesseractConfig.cs} (86%) diff --git a/.gitignore b/.gitignore index a06144dcb..bba535498 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ dotnet/.config *.user *.userosscache *.sln.docstates +webapi/data/*.json +__dev/ # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs diff --git a/CopilotChat.sln.DotSettings b/CopilotChat.sln.DotSettings index e0cd1b828..d1a28aa42 100644 --- a/CopilotChat.sln.DotSettings +++ b/CopilotChat.sln.DotSettings @@ -289,6 +289,7 @@ public void It$SOMENAME$() True True True + True True True True diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 000000000..197770b35 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,51 @@ + + + + 12 + + + LatestMajor + + + enable + + + enable + + false + All + CA1812,CA2234,CS1570,CS1572,CS1573,CS1574,CA1056,CA1716,CA1724,SKEXP0003,SKEXP0011,SKEXP0021,SKEXP0026,SKEXP0042,SKEXP0050,SKEXP0052,SKEXP0053,SKEXP0060,KMEXP02,SKEXP0040 + true + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 000000000..f4b7b2d84 --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,102 @@ + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + \ No newline at end of file diff --git a/integration-tests/ChatCopilotIntegrationTest.cs b/integration-tests/ChatCopilotIntegrationTest.cs index 44e1fff5a..719e01c8f 100644 --- a/integration-tests/ChatCopilotIntegrationTest.cs +++ b/integration-tests/ChatCopilotIntegrationTest.cs @@ -1,10 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Linq; -using System.Net.Http; using System.Net.Http.Headers; -using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Identity.Client; using Xunit; @@ -15,6 +11,7 @@ namespace ChatCopilotIntegrationTests; /// Base class for Chat Copilot integration tests /// [Trait("Category", "Integration Tests")] +#pragma warning disable CA1051 public abstract class ChatCopilotIntegrationTest : IDisposable { protected const string BaseUrlSettingName = "BaseServerUrl"; @@ -76,7 +73,7 @@ protected async Task GetUserTokenByPasswordAsync() string? scopeString = this.Configuration[ScopesSettingName]; Assert.NotNull(scopeString); - string[] scopes = scopeString.Split(new char[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries); + string[] scopes = scopeString.Split([',', ' '], StringSplitOptions.RemoveEmptyEntries); var accounts = await app.GetAccountsAsync(); diff --git a/integration-tests/ChatCopilotIntegrationTests.csproj b/integration-tests/ChatCopilotIntegrationTests.csproj index 419e81b4f..ebca6076c 100644 --- a/integration-tests/ChatCopilotIntegrationTests.csproj +++ b/integration-tests/ChatCopilotIntegrationTests.csproj @@ -2,50 +2,35 @@ net8.0 - enable - false true 81136671-a63f-4f20-bd0e-a65b2561999a - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - - - - PreserveNewest diff --git a/integration-tests/ChatTests.cs b/integration-tests/ChatTests.cs index aac84f509..c03a2dbd1 100644 --- a/integration-tests/ChatTests.cs +++ b/integration-tests/ChatTests.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Net.Http; using System.Net.Http.Json; using System.Text.Json; using CopilotChat.WebApi.Models.Request; @@ -13,6 +11,8 @@ namespace ChatCopilotIntegrationTests; public class ChatTests : ChatCopilotIntegrationTest { + private static readonly JsonSerializerOptions jsOpts = new() { PropertyNameCaseInsensitive = true }; + [Fact] public async void ChatMessagePostSucceedsWithValidInput() { @@ -24,7 +24,7 @@ public async void ChatMessagePostSucceedsWithValidInput() response.EnsureSuccessStatusCode(); var contentStream = await response.Content.ReadAsStreamAsync(); - var createChatResponse = await JsonSerializer.DeserializeAsync(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + var createChatResponse = await JsonSerializer.DeserializeAsync(contentStream, jsOpts); Assert.NotNull(createChatResponse); // Ask something to the bot @@ -37,12 +37,12 @@ public async void ChatMessagePostSucceedsWithValidInput() response.EnsureSuccessStatusCode(); contentStream = await response.Content.ReadAsStreamAsync(); - var askResult = await JsonSerializer.DeserializeAsync(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + var askResult = await JsonSerializer.DeserializeAsync(contentStream, jsOpts); Assert.NotNull(askResult); Assert.False(string.IsNullOrEmpty(askResult.Value)); // Clean up - response = await this.HTTPClient.DeleteAsync($"chats/{createChatResponse.ChatSession.Id}"); + response = await this.HTTPClient.DeleteAsync($"chats/{createChatResponse.ChatSession.Id}").ConfigureAwait(false); response.EnsureSuccessStatusCode(); } } diff --git a/integration-tests/HealthzTests.cs b/integration-tests/HealthzTests.cs index d35a4cc9f..d3c77ba06 100644 --- a/integration-tests/HealthzTests.cs +++ b/integration-tests/HealthzTests.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Net.Http; using Xunit; namespace ChatCopilotIntegrationTests; diff --git a/integration-tests/ServiceInfoTests.cs b/integration-tests/ServiceInfoTests.cs index d71aac6a3..1051b2060 100644 --- a/integration-tests/ServiceInfoTests.cs +++ b/integration-tests/ServiceInfoTests.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Net.Http; using System.Text.Json; using CopilotChat.WebApi.Models.Response; using CopilotChat.WebApi.Options; @@ -10,6 +9,8 @@ namespace ChatCopilotIntegrationTests; public class ServiceInfoTests : ChatCopilotIntegrationTest { + private static readonly JsonSerializerOptions jsonOptions = new() { PropertyNameCaseInsensitive = true }; + [Fact] public async void GetServiceInfo() { @@ -19,7 +20,7 @@ public async void GetServiceInfo() response.EnsureSuccessStatusCode(); var contentStream = await response.Content.ReadAsStreamAsync(); - var objectFromResponse = await JsonSerializer.DeserializeAsync(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + var objectFromResponse = await JsonSerializer.DeserializeAsync(contentStream, jsonOptions); Assert.NotNull(objectFromResponse); Assert.False(string.IsNullOrEmpty(objectFromResponse.MemoryStore.SelectedType)); @@ -33,7 +34,7 @@ public async void GetAuthConfig() response.EnsureSuccessStatusCode(); var contentStream = await response.Content.ReadAsStreamAsync(); - var objectFromResponse = await JsonSerializer.DeserializeAsync(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + var objectFromResponse = await JsonSerializer.DeserializeAsync(contentStream, jsonOptions); Assert.NotNull(objectFromResponse); Assert.Equal(ChatAuthenticationOptions.AuthenticationType.AzureAd.ToString(), objectFromResponse.AuthType); diff --git a/integration-tests/SpeechTokenTests.cs b/integration-tests/SpeechTokenTests.cs index 337ddc234..b3fbd6a37 100644 --- a/integration-tests/SpeechTokenTests.cs +++ b/integration-tests/SpeechTokenTests.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Net.Http; using System.Text.Json; using CopilotChat.WebApi.Models.Response; using Xunit; @@ -9,6 +8,8 @@ namespace ChatCopilotIntegrationTests; public class SpeechTokenTests : ChatCopilotIntegrationTest { + private static readonly JsonSerializerOptions jsonOpts = new() { PropertyNameCaseInsensitive = true }; + [Fact] public async void GetSpeechToken() { @@ -18,7 +19,7 @@ public async void GetSpeechToken() response.EnsureSuccessStatusCode(); var contentStream = await response.Content.ReadAsStreamAsync(); - var speechTokenResponse = await JsonSerializer.DeserializeAsync(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + var speechTokenResponse = await JsonSerializer.DeserializeAsync(contentStream, jsonOpts); Assert.NotNull(speechTokenResponse); Assert.True((speechTokenResponse.IsSuccess == true && !string.IsNullOrEmpty(speechTokenResponse.Token)) || diff --git a/integration-tests/StaticFiles.cs b/integration-tests/StaticFiles.cs index 5c98348b1..42c160ff1 100644 --- a/integration-tests/StaticFiles.cs +++ b/integration-tests/StaticFiles.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Net.Http; using Xunit; namespace ChatCopilotIntegrationTests; diff --git a/memorypipeline/CopilotChatMemoryPipeline.csproj b/memorypipeline/CopilotChatMemoryPipeline.csproj index 3efbeb226..b0133d9aa 100644 --- a/memorypipeline/CopilotChatMemoryPipeline.csproj +++ b/memorypipeline/CopilotChatMemoryPipeline.csproj @@ -3,9 +3,6 @@ CopilotChat.MemoryPipeline net8.0 - LatestMajor - disable - enable 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 @@ -14,25 +11,9 @@ - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + + diff --git a/memorypipeline/Program.cs b/memorypipeline/Program.cs index 8e69548b4..3c4c50a02 100644 --- a/memorypipeline/Program.cs +++ b/memorypipeline/Program.cs @@ -1,11 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using CopilotChat.Shared; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Microsoft.KernelMemory; using Microsoft.KernelMemory.Diagnostics; @@ -18,7 +13,6 @@ IKernelMemory memory = new KernelMemoryBuilder(builder.Services) .FromAppSettings() - .WithCustomOcr(builder.Configuration) .Build(); builder.Services.AddSingleton(memory); diff --git a/plugins/shared/PluginShared.csproj b/plugins/shared/PluginShared.csproj index 609406c1f..1c648a957 100644 --- a/plugins/shared/PluginShared.csproj +++ b/plugins/shared/PluginShared.csproj @@ -2,27 +2,7 @@ net6.0 - enable Plugins.PluginShared - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - \ No newline at end of file diff --git a/plugins/web-searcher/PluginEndpoint.cs b/plugins/web-searcher/PluginEndpoint.cs index 589ad59cd..020450683 100644 --- a/plugins/web-searcher/PluginEndpoint.cs +++ b/plugins/web-searcher/PluginEndpoint.cs @@ -1,18 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.IO; -using System.Linq; using System.Net; -using System.Net.Http; using System.Text.Json; -using System.Threading.Tasks; using Microsoft.AspNetCore.WebUtilities; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Azure.WebJobs.Extensions.OpenApi.Core.Attributes; using Microsoft.Azure.WebJobs.Extensions.OpenApi.Core.Enums; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; using Microsoft.OpenApi.Models; using Plugins.PluginShared; using Plugins.WebSearcher.Models; @@ -99,7 +95,7 @@ public async Task IconAsync( /// /// The http request data. /// A string representing the search result. - [OpenApiOperation(operationId: "Search", tags: new[] { "WebSearchfunction" }, Description = "Searches the web for the given query.")] + [OpenApiOperation(operationId: "Search", tags: ["WebSearchfunction"], Description = "Searches the web for the given query.")] [OpenApiSecurity("function_key", SecuritySchemeType.ApiKey, Name = "x-functions-key", In = OpenApiSecurityLocationType.Header)] [OpenApiParameter(name: "Query", In = ParameterLocation.Query, Required = true, Type = typeof(string), Description = "The query")] [OpenApiParameter(name: "NumResults", In = ParameterLocation.Query, Required = true, Type = typeof(int), Description = "The maximum number of results to return")] @@ -110,26 +106,23 @@ public async Task IconAsync( [Function("WebSearch")] public async Task WebSearchAsync([HttpTrigger(AuthorizationLevel.Function, "get", Route = "search")] HttpRequestData req) { - var queries = QueryHelpers.ParseQuery(req.Url.Query); - var query = queries.ContainsKey("Query") ? queries["Query"].ToString() : string.Empty; + var queryParams = QueryHelpers.ParseQuery(req.Url.Query); + + string query = queryParams.TryGetValue("Query", out StringValues q1) ? q1.ToString() : string.Empty; + int numResults = queryParams.TryGetValue("NumResults", out StringValues q2) && int.TryParse(q2, out int q2Value) ? q2Value : 0; + int offset = queryParams.TryGetValue("Offset", out StringValues q3) && int.TryParse(q3, out var q3Value) ? q3Value : 0; + string site = queryParams.TryGetValue("Site", out StringValues q4) ? q4.ToString() : string.Empty; + if (string.IsNullOrWhiteSpace(query)) { return await this.CreateBadRequestResponseAsync(req, "Empty query."); } - var numResults = queries.ContainsKey("NumResults") ? int.Parse(queries["NumResults"]!) : 0; if (numResults <= 0) { return await this.CreateBadRequestResponseAsync(req, "Invalid number of results."); } - int offset = 0; - if (queries.TryGetValue("Offset", out var offsetValue)) - { - int.TryParse(offsetValue, out offset); - } - - var site = queries.ContainsKey("Site") ? queries["Site"].ToString() : string.Empty; if (string.IsNullOrWhiteSpace(site)) { this._logger.LogDebug("Searching the web for '{0}'", query); diff --git a/plugins/web-searcher/WebSearcher.csproj b/plugins/web-searcher/WebSearcher.csproj index f7612702c..6864f8e40 100644 --- a/plugins/web-searcher/WebSearcher.csproj +++ b/plugins/web-searcher/WebSearcher.csproj @@ -3,36 +3,19 @@ net8.0 v4 Exe - enable Plugins.WebSearcher - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + - + + + + + @@ -47,6 +30,7 @@ PreserveNewest + diff --git a/shared/ConfigurationBuilderExtensions.cs b/shared/ConfigurationBuilderExtensions.cs index b0006179a..1bd786495 100644 --- a/shared/ConfigurationBuilderExtensions.cs +++ b/shared/ConfigurationBuilderExtensions.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.IO; using System.Reflection; using Microsoft.Extensions.Configuration; diff --git a/shared/CopilotChatShared.csproj b/shared/CopilotChatShared.csproj index 32324f3dd..a89a74758 100644 --- a/shared/CopilotChatShared.csproj +++ b/shared/CopilotChatShared.csproj @@ -3,33 +3,22 @@ CopilotChat.Shared net8.0 - LatestMajor - disable - enable - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - + + + + + + + + + + + + + diff --git a/shared/MemoryClientBuilderExtensions.cs b/shared/MemoryClientBuilderExtensions.cs deleted file mode 100644 index 3a54bd577..000000000 --- a/shared/MemoryClientBuilderExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using CopilotChat.Shared.Ocr; -using Microsoft.Extensions.Configuration; -using Microsoft.KernelMemory; - -namespace CopilotChat.Shared; - -/// -/// Dependency injection for kernel memory using custom OCR configuration defined in appsettings.json -/// -public static class MemoryClientBuilderExtensions -{ - public static IKernelMemoryBuilder WithCustomOcr(this IKernelMemoryBuilder builder, IConfiguration configuration) - { - var ocrEngine = configuration.CreateCustomOcr(); - - if (ocrEngine is not null) - { - builder.WithCustomImageOcr(ocrEngine); - } - - return builder; - } -} diff --git a/shared/Ocr/ConfigurationExtensions.cs b/shared/Ocr/ConfigurationExtensions.cs deleted file mode 100644 index c53743a42..000000000 --- a/shared/Ocr/ConfigurationExtensions.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using CopilotChat.Shared.Ocr.Tesseract; -using Microsoft.Extensions.Configuration; -using Microsoft.KernelMemory.DataFormats; - -namespace CopilotChat.Shared.Ocr; - -/// -/// Dependency injection for kernel memory using configuration defined in appsettings.json -/// -public static class ConfigurationExtensions -{ - private const string ConfigOcrType = "ImageOcrType"; - - public static IOcrEngine? CreateCustomOcr(this IConfiguration configuration) - { - var ocrType = configuration.GetSection($"{MemoryConfiguration.KernelMemorySection}:{ConfigOcrType}").Value ?? string.Empty; - switch (ocrType) - { - case string x when x.Equals(TesseractOptions.SectionName, StringComparison.OrdinalIgnoreCase): - var tesseractOptions = - configuration - .GetSection($"{MemoryConfiguration.KernelMemorySection}:{MemoryConfiguration.ServicesSection}:{TesseractOptions.SectionName}") - .Get(); - - if (tesseractOptions == null) - { - throw new ArgumentNullException($"Missing configuration for {ConfigOcrType}: {ocrType}"); - } - - return new TesseractOcrEngine(tesseractOptions); - - default: // Allow for fall-through for standard OCR settings - break; - } - - return null; - } -} diff --git a/shared/Ocr/Tesseract/TesseractOptions.cs b/shared/Ocr/Tesseract/TesseractConfig.cs similarity index 86% rename from shared/Ocr/Tesseract/TesseractOptions.cs rename to shared/Ocr/Tesseract/TesseractConfig.cs index 87c318227..361d1fb86 100644 --- a/shared/Ocr/Tesseract/TesseractOptions.cs +++ b/shared/Ocr/Tesseract/TesseractConfig.cs @@ -7,10 +7,8 @@ namespace CopilotChat.Shared.Ocr.Tesseract; /// /// Configuration options for Tesseract OCR support. /// -public sealed class TesseractOptions +public sealed class TesseractConfig { - public const string SectionName = "Tesseract"; - /// /// The file path where the Tesseract language file is stored (e.g. "./data") /// diff --git a/shared/Ocr/Tesseract/TesseractOcrEngine.cs b/shared/Ocr/Tesseract/TesseractOcrEngine.cs index 75881c90c..6b5d7c7ae 100644 --- a/shared/Ocr/Tesseract/TesseractOcrEngine.cs +++ b/shared/Ocr/Tesseract/TesseractOcrEngine.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.IO; -using System.Threading; -using System.Threading.Tasks; using Microsoft.KernelMemory.DataFormats; using Tesseract; @@ -11,16 +8,21 @@ namespace CopilotChat.Shared.Ocr.Tesseract; /// /// Wrapper for the TesseractEngine within the Tesseract OCR library. /// -public class TesseractOcrEngine : IOcrEngine +public sealed class TesseractOcrEngine : IOcrEngine, IDisposable { private readonly TesseractEngine _engine; /// /// Creates a new instance of the TesseractEngineWrapper passing in a valid TesseractEngine. /// - public TesseractOcrEngine(TesseractOptions tesseractOptions) + public TesseractOcrEngine(TesseractConfig tesseractConfig) { - this._engine = new TesseractEngine(tesseractOptions.FilePath, tesseractOptions.Language); + if (!Directory.Exists(tesseractConfig.FilePath)) + { + throw new DirectoryNotFoundException($"{tesseractConfig.FilePath} dir not found"); + } + + this._engine = new TesseractEngine(tesseractConfig.FilePath, tesseractConfig.Language); } /// @@ -28,7 +30,7 @@ public async Task ExtractTextFromImageAsync(Stream imageContent, Cancell { await using (var imgStream = new MemoryStream()) { - await imageContent.CopyToAsync(imgStream); + await imageContent.CopyToAsync(imgStream, cancellationToken); imgStream.Position = 0; using var img = Pix.LoadFromMemory(imgStream.ToArray()); @@ -37,4 +39,10 @@ public async Task ExtractTextFromImageAsync(Stream imageContent, Cancell return page.GetText(); } } + + /// + public void Dispose() + { + this._engine.Dispose(); + } } diff --git a/shared/ServiceConfiguration.cs b/shared/ServiceConfiguration.cs index 3507469cb..313de466f 100644 --- a/shared/ServiceConfiguration.cs +++ b/shared/ServiceConfiguration.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; +using CopilotChat.Shared.Ocr.Tesseract; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.KernelMemory; using Microsoft.KernelMemory.AI; +using Microsoft.KernelMemory.AI.Ollama; +using Microsoft.KernelMemory.AI.OpenAI; using Microsoft.KernelMemory.DocumentStorage.DevTools; +using Microsoft.KernelMemory.MemoryDb.SQLServer; using Microsoft.KernelMemory.MemoryStorage; using Microsoft.KernelMemory.MemoryStorage.DevTools; using Microsoft.KernelMemory.Pipeline.Queue.DevTools; @@ -24,9 +26,6 @@ internal sealed class ServiceConfiguration // appsettings.json root node name private const string ConfigRoot = "KernelMemory"; - // ASP.NET env var - private const string AspnetEnvVar = "ASPNETCORE_ENVIRONMENT"; - // OpenAI env var private const string OpenAIEnvVar = "OPENAI_API_KEY"; @@ -74,7 +73,7 @@ private IKernelMemoryBuilder BuildUsingConfiguration(IKernelMemoryBuilder builde } // Required by ctors expecting KernelMemoryConfig via DI - builder.AddSingleton(this._memoryConfiguration); + builder.AddSingleton(this._memoryConfiguration); this.ConfigureMimeTypeDetectionDependency(builder); @@ -125,15 +124,18 @@ private void ConfigureQueueDependency(IKernelMemoryBuilder builder) { switch (this._memoryConfiguration.DataIngestion.DistributedOrchestration.QueueType) { - case string y1 when y1.Equals("AzureQueue", StringComparison.OrdinalIgnoreCase): - case string y2 when y2.Equals("AzureQueues", StringComparison.OrdinalIgnoreCase): - // Check 2 keys for backward compatibility - builder.Services.AddAzureQueuesOrchestration(this.GetServiceConfig("AzureQueue")); + case string x when x.Equals("AzureQueue", StringComparison.Ordinal): + throw new ConfigurationException("The configuration key 'AzureQueue' has been deprecated. Please use 'AzureQueues' instead.'"); + + case string x when x.Equals("RabbitMq", StringComparison.Ordinal): + throw new ConfigurationException("The configuration key 'RabbitMq' has been deprecated. Please use 'RabbitMQ' instead.'"); + + case string x when x.Equals("AzureQueues", StringComparison.OrdinalIgnoreCase): + builder.Services.AddAzureQueuesOrchestration(this.GetServiceConfig("AzureQueues")); break; case string y when y.Equals("RabbitMQ", StringComparison.OrdinalIgnoreCase): - // Check 2 keys for backward compatibility - builder.Services.AddRabbitMQOrchestration(this.GetServiceConfig("RabbitMq")); + builder.Services.AddRabbitMQOrchestration(this.GetServiceConfig("RabbitMQ")); break; case string y when y.Equals("SimpleQueues", StringComparison.OrdinalIgnoreCase): @@ -151,15 +153,15 @@ private void ConfigureStorageDependency(IKernelMemoryBuilder builder) { switch (this._memoryConfiguration.DocumentStorageType) { - case string x1 when x1.Equals("AzureBlob", StringComparison.OrdinalIgnoreCase): - case string x2 when x2.Equals("AzureBlobs", StringComparison.OrdinalIgnoreCase): - // Check 2 keys for backward compatibility - builder.WithAzureBlobsDocumentStorage(this.GetServiceConfig("AzureBlobs") - ?? this.GetServiceConfig("AzureBlob")); + case string x when x.Equals("AzureBlob", StringComparison.OrdinalIgnoreCase): + throw new ConfigurationException("The configuration key 'AzureBlob' has been deprecated. Please use 'AzureBlobs' instead.'"); + + case string x when x.Equals("AzureBlobs", StringComparison.OrdinalIgnoreCase): + builder.Services.AddAzureBlobsAsDocumentStorage(this.GetServiceConfig("AzureBlobs")); break; case string x when x.Equals("SimpleFileStorage", StringComparison.OrdinalIgnoreCase): - builder.WithSimpleFileStorage(this.GetServiceConfig("SimpleFileStorage")); + builder.Services.AddSimpleFileStorageAsDocumentStorage(this.GetServiceConfig("SimpleFileStorage")); break; default: @@ -201,7 +203,9 @@ private void ConfigureIngestionEmbeddingGenerators(IKernelMemoryBuilder builder) case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase): { var instance = this.GetServiceInstance(builder, - s => s.AddAzureOpenAIEmbeddingGeneration(this.GetServiceConfig("AzureOpenAIEmbedding"))); + s => s.AddAzureOpenAIEmbeddingGeneration( + config: this.GetServiceConfig("AzureOpenAIEmbedding"), + textTokenizer: new GPT4oTokenizer())); builder.AddIngestionEmbeddingGenerator(instance); break; } @@ -209,7 +213,19 @@ private void ConfigureIngestionEmbeddingGenerators(IKernelMemoryBuilder builder) case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): { var instance = this.GetServiceInstance(builder, - s => s.AddOpenAITextEmbeddingGeneration(this.GetServiceConfig("OpenAI"))); + s => s.AddOpenAITextEmbeddingGeneration( + config: this.GetServiceConfig("OpenAI"), + textTokenizer: new GPT4oTokenizer())); + builder.AddIngestionEmbeddingGenerator(instance); + break; + } + + case string x when x.Equals("Ollama", StringComparison.OrdinalIgnoreCase): + { + var instance = this.GetServiceInstance(builder, + s => s.AddOllamaTextEmbeddingGeneration( + config: this.GetServiceConfig("Ollama"), + textTokenizer: new GPT4oTokenizer())); builder.AddIngestionEmbeddingGenerator(instance); break; } @@ -246,15 +262,6 @@ private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder) break; } - case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase): - { - var instance = this.GetServiceInstance(builder, - s => s.AddQdrantAsMemoryDb(this.GetServiceConfig("Qdrant")) - ); - builder.AddIngestionMemoryDb(instance); - break; - } - case string x when x.Equals("Postgres", StringComparison.OrdinalIgnoreCase): { var instance = this.GetServiceInstance(builder, @@ -264,10 +271,10 @@ private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder) break; } - case string x when x.Equals("Redis", StringComparison.OrdinalIgnoreCase): + case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase): { var instance = this.GetServiceInstance(builder, - s => s.AddRedisAsMemoryDb(this.GetServiceConfig("Redis")) + s => s.AddQdrantAsMemoryDb(this.GetServiceConfig("Qdrant")) ); builder.AddIngestionMemoryDb(instance); break; @@ -282,10 +289,10 @@ private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder) break; } - case string x when x.Equals("SimpleTextDb", StringComparison.OrdinalIgnoreCase): + case string x when x.Equals("SqlServer", StringComparison.OrdinalIgnoreCase): { var instance = this.GetServiceInstance(builder, - s => s.AddSimpleTextDbAsMemoryDb(this.GetServiceConfig("SimpleTextDb")) + s => s.AddSqlServerAsMemoryDb(this.GetServiceConfig("SqlServer")) ); builder.AddIngestionMemoryDb(instance); break; @@ -307,11 +314,21 @@ private void ConfigureRetrievalEmbeddingGenerator(IKernelMemoryBuilder builder) { case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase): case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase): - builder.Services.AddAzureOpenAIEmbeddingGeneration(this.GetServiceConfig("AzureOpenAIEmbedding")); + builder.Services.AddAzureOpenAIEmbeddingGeneration( + config: this.GetServiceConfig("AzureOpenAIEmbedding"), + textTokenizer: new GPT4oTokenizer()); break; case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): - builder.Services.AddOpenAITextEmbeddingGeneration(this.GetServiceConfig("OpenAI")); + builder.Services.AddOpenAITextEmbeddingGeneration( + config: this.GetServiceConfig("OpenAI"), + textTokenizer: new GPT4oTokenizer()); + break; + + case string x when x.Equals("Ollama", StringComparison.OrdinalIgnoreCase): + builder.Services.AddOllamaTextEmbeddingGeneration( + config: this.GetServiceConfig("Ollama"), + textTokenizer: new GPT4oTokenizer()); break; default: @@ -329,24 +346,20 @@ private void ConfigureRetrievalMemoryDb(IKernelMemoryBuilder builder) builder.Services.AddAzureAISearchAsMemoryDb(this.GetServiceConfig("AzureAISearch")); break; - case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase): - builder.Services.AddQdrantAsMemoryDb(this.GetServiceConfig("Qdrant")); - break; - case string x when x.Equals("Postgres", StringComparison.OrdinalIgnoreCase): builder.Services.AddPostgresAsMemoryDb(this.GetServiceConfig("Postgres")); break; - case string x when x.Equals("Redis", StringComparison.OrdinalIgnoreCase): - builder.Services.AddRedisAsMemoryDb(this.GetServiceConfig("Redis")); + case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase): + builder.Services.AddQdrantAsMemoryDb(this.GetServiceConfig("Qdrant")); break; case string x when x.Equals("SimpleVectorDb", StringComparison.OrdinalIgnoreCase): builder.Services.AddSimpleVectorDbAsMemoryDb(this.GetServiceConfig("SimpleVectorDb")); break; - case string x when x.Equals("SimpleTextDb", StringComparison.OrdinalIgnoreCase): - builder.Services.AddSimpleTextDbAsMemoryDb(this.GetServiceConfig("SimpleTextDb")); + case string x when x.Equals("SqlServer", StringComparison.OrdinalIgnoreCase): + builder.Services.AddSqlServerAsMemoryDb(this.GetServiceConfig("SqlServer")); break; default: @@ -362,11 +375,21 @@ private void ConfigureTextGenerator(IKernelMemoryBuilder builder) { case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase): case string y when y.Equals("AzureOpenAIText", StringComparison.OrdinalIgnoreCase): - builder.Services.AddAzureOpenAITextGeneration(this.GetServiceConfig("AzureOpenAIText")); + builder.Services.AddAzureOpenAITextGeneration( + config: this.GetServiceConfig("AzureOpenAIText"), + textTokenizer: new GPT4oTokenizer()); break; case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase): - builder.Services.AddOpenAITextGeneration(this.GetServiceConfig("OpenAI")); + builder.Services.AddOpenAITextGeneration( + config: this.GetServiceConfig("OpenAI"), + textTokenizer: new GPT4oTokenizer()); + break; + + case string x when x.Equals("Ollama", StringComparison.OrdinalIgnoreCase): + builder.Services.AddOllamaTextGeneration( + config: this.GetServiceConfig("Ollama"), + textTokenizer: new GPT4oTokenizer()); break; default: @@ -388,6 +411,11 @@ private void ConfigureImageOCR(IKernelMemoryBuilder builder) builder.Services.AddAzureAIDocIntel(this.GetServiceConfig("AzureAIDocIntel")); break; + case string x when x.Equals("Tesseract", StringComparison.OrdinalIgnoreCase): + builder.Services.AddSingleton(this.GetServiceConfig("Tesseract")); + builder.WithCustomImageOcr(); + break; + default: // NOOP - allow custom implementations, via WithCustomImageOCR() break; diff --git a/tools/importdocument/ImportDocument.csproj b/tools/importdocument/ImportDocument.csproj index d82421011..b2b3eacc6 100644 --- a/tools/importdocument/ImportDocument.csproj +++ b/tools/importdocument/ImportDocument.csproj @@ -2,11 +2,6 @@ Exe net6.0 - LatestMajor - disable - enable - 10 - false @@ -16,26 +11,10 @@ - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - + + + + diff --git a/tools/importdocument/Program.cs b/tools/importdocument/Program.cs index 3d9c89c8a..5bbddc5e2 100644 --- a/tools/importdocument/Program.cs +++ b/tools/importdocument/Program.cs @@ -1,13 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.CommandLine; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Identity.Client; namespace ImportDocument; diff --git a/webapi/Auth/AuthInfo.cs b/webapi/Auth/AuthInfo.cs index 5a0e00ec0..bd7c689bc 100644 --- a/webapi/Auth/AuthInfo.cs +++ b/webapi/Auth/AuthInfo.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using Azure.Identity; -using Microsoft.AspNetCore.Http; using Microsoft.Identity.Web; namespace CopilotChat.WebApi.Auth; diff --git a/webapi/Auth/ChatParticipantAuthorizationHandler.cs b/webapi/Auth/ChatParticipantAuthorizationHandler.cs index 333503cdf..add5a8d13 100644 --- a/webapi/Auth/ChatParticipantAuthorizationHandler.cs +++ b/webapi/Auth/ChatParticipantAuthorizationHandler.cs @@ -1,10 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Threading.Tasks; +using Azure.Identity; using CopilotChat.WebApi.Storage; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Routing; namespace CopilotChat.WebApi.Auth; @@ -61,7 +59,7 @@ protected override async Task HandleRequirementAsync( context.Succeed(requirement); } - catch (Azure.Identity.CredentialUnavailableException ex) + catch (CredentialUnavailableException ex) { context.Fail(new AuthorizationFailureReason(this, ex.Message)); } diff --git a/webapi/Auth/PassThroughAuthenticationHandler.cs b/webapi/Auth/PassThroughAuthenticationHandler.cs index 8b6f2a1c6..661e79bf6 100644 --- a/webapi/Auth/PassThroughAuthenticationHandler.cs +++ b/webapi/Auth/PassThroughAuthenticationHandler.cs @@ -2,9 +2,7 @@ using System.Security.Claims; using System.Text.Encodings.Web; -using System.Threading.Tasks; using Microsoft.AspNetCore.Authentication; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Identity.Web; diff --git a/webapi/Controllers/ChatArchiveController.cs b/webapi/Controllers/ChatArchiveController.cs index a7e48f4e7..7fcaf26c2 100644 --- a/webapi/Controllers/ChatArchiveController.cs +++ b/webapi/Controllers/ChatArchiveController.cs @@ -1,10 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Models.Response; @@ -12,9 +7,7 @@ using CopilotChat.WebApi.Options; using CopilotChat.WebApi.Storage; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; diff --git a/webapi/Controllers/ChatController.cs b/webapi/Controllers/ChatController.cs index f1340e215..49e177b6b 100644 --- a/webapi/Controllers/ChatController.cs +++ b/webapi/Controllers/ChatController.cs @@ -1,17 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; using System.Net.Http.Headers; using System.Reflection; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Models.Request; @@ -21,11 +14,8 @@ using CopilotChat.WebApi.Plugins.Chat; using CopilotChat.WebApi.Services; using CopilotChat.WebApi.Storage; -using CopilotChat.WebApi.Utilities; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Graph; using Microsoft.SemanticKernel; @@ -240,7 +230,7 @@ private async Task RegisterGithubPluginAsync(Kernel kernel, string githubAuthHea BearerAuthenticationProvider authenticationProvider = new(() => Task.FromResult(githubAuthHeader)); await kernel.ImportPluginFromOpenApiAsync( pluginName: "GitHubPlugin", - filePath: GetPluginFullPath("GitHubPlugin/openapi.json"), + filePath: GetPluginFullPath("OpenApi/GitHubPlugin/openapi.json"), new OpenApiFunctionExecutionParameters { AuthCallback = authenticationProvider.AuthenticateRequestAsync, @@ -286,6 +276,26 @@ private Task RegisterMicrosoftGraphOBOPlugins(Kernel kernel, string graphOboAuth return Task.CompletedTask; } + /// + /// Create a Microsoft Graph service client. + /// + /// The delegate to authenticate the request. + private GraphServiceClient CreateGraphServiceClient(AuthenticateRequestAsyncDelegate authenticateRequestAsyncDelegate) + { + MsGraphClientLoggingHandler graphLoggingHandler = new(this._logger); + this._disposables.Add(graphLoggingHandler); + + IList graphMiddlewareHandlers = + GraphClientFactory.CreateDefaultHandlers(new DelegateAuthenticationProvider(authenticateRequestAsyncDelegate)); + graphMiddlewareHandlers.Add(graphLoggingHandler); + + HttpClient graphHttpClient = GraphClientFactory.Create(graphMiddlewareHandlers); + this._disposables.Add(graphHttpClient); + + GraphServiceClient graphServiceClient = new(graphHttpClient); + return graphServiceClient; + } + private IEnumerable RegisterCustomPlugins(Kernel kernel, object? customPluginsString, Dictionary authHeaders) { CustomPlugin[]? customPlugins = JsonSerializer.Deserialize(customPluginsString!.ToString()!); @@ -296,6 +306,9 @@ private IEnumerable RegisterCustomPlugins(Kernel kernel, object? customPlu { if (authHeaders.TryGetValue(plugin.AuthHeaderTag.ToUpperInvariant(), out string? pluginAuthValue)) { + this._logger.LogError("ChatGPT plugins are deprecated. Skipping plugin: {0}", plugin.NameForHuman); + + /* // Register the ChatGPT plugin with the kernel. this._logger.LogInformation("Enabling {0} plugin.", plugin.NameForHuman); @@ -318,6 +331,7 @@ Task AuthCallback(HttpRequestMessage request, string _, OpenAIAuthenticationConf IgnoreNonCompliantErrors = true, AuthCallback = requiresAuth ? AuthCallback : null }); + */ } } } @@ -325,34 +339,19 @@ Task AuthCallback(HttpRequestMessage request, string _, OpenAIAuthenticationConf { this._logger.LogDebug("Failed to deserialize custom plugin details: {0}", customPluginsString); } - } - /// - /// Create a Microsoft Graph service client. - /// - /// The delegate to authenticate the request. - private GraphServiceClient CreateGraphServiceClient(AuthenticateRequestAsyncDelegate authenticateRequestAsyncDelegate) - { - MsGraphClientLoggingHandler graphLoggingHandler = new(this._logger); - this._disposables.Add(graphLoggingHandler); - - IList graphMiddlewareHandlers = - GraphClientFactory.CreateDefaultHandlers(new DelegateAuthenticationProvider(authenticateRequestAsyncDelegate)); - graphMiddlewareHandlers.Add(graphLoggingHandler); - - HttpClient graphHttpClient = GraphClientFactory.Create(graphMiddlewareHandlers); - this._disposables.Add(graphHttpClient); - - GraphServiceClient graphServiceClient = new(graphHttpClient); - return graphServiceClient; + yield return Task.CompletedTask; } - private async Task RegisterHostedFunctionsAsync(Kernel kernel, HashSet enabledPlugins) + private Task RegisterHostedFunctionsAsync(Kernel kernel, HashSet enabledPlugins) { foreach (string enabledPlugin in enabledPlugins) { if (this._plugins.TryGetValue(enabledPlugin, out Plugin? plugin)) { + this._logger.LogError("ChatGPT plugins are deprecated. Skipping plugin {0}", enabledPlugin); + + /* this._logger.LogDebug("Enabling hosted plugin {0}.", plugin.Name); Task AuthCallback(HttpRequestMessage request, string _, OpenAIAuthenticationConfig __, CancellationToken ___ = default) @@ -372,6 +371,7 @@ await kernel.ImportPluginFromOpenAIAsync( IgnoreNonCompliantErrors = true, AuthCallback = AuthCallback }); + */ } else { @@ -379,7 +379,7 @@ await kernel.ImportPluginFromOpenAIAsync( } } - return; + return Task.CompletedTask; } private static KernelArguments GetContextVariables(Ask ask, IAuthInfo authInfo, string chatId) @@ -496,13 +496,4 @@ public async Task GraphClientAuthenticateRequestAsync(HttpRequestMessage request { await this.AuthenticateRequestAsync(request); } - - /// - /// Applies the token to the provided HTTP request message. - /// - /// The HTTP request message. - public async Task OpenAIAuthenticateRequestAsync(HttpRequestMessage request, string pluginName, OpenAIAuthenticationConfig openAIAuthConfig, CancellationToken cancellationToken = default) - { - await this.AuthenticateRequestAsync(request, cancellationToken); - } } diff --git a/webapi/Controllers/ChatHistoryController.cs b/webapi/Controllers/ChatHistoryController.cs index 68353c466..111b256b0 100644 --- a/webapi/Controllers/ChatHistoryController.cs +++ b/webapi/Controllers/ChatHistoryController.cs @@ -1,10 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Hubs; @@ -15,10 +10,8 @@ using CopilotChat.WebApi.Plugins.Utils; using CopilotChat.WebApi.Storage; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; diff --git a/webapi/Controllers/ChatMemoryController.cs b/webapi/Controllers/ChatMemoryController.cs index be1f7b8f3..07c439a15 100644 --- a/webapi/Controllers/ChatMemoryController.cs +++ b/webapi/Controllers/ChatMemoryController.cs @@ -1,18 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Models.Request; using CopilotChat.WebApi.Options; using CopilotChat.WebApi.Storage; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; diff --git a/webapi/Controllers/ChatParticipantController.cs b/webapi/Controllers/ChatParticipantController.cs index cecc4ae3f..cfd7b78bc 100644 --- a/webapi/Controllers/ChatParticipantController.cs +++ b/webapi/Controllers/ChatParticipantController.cs @@ -1,16 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Models.Storage; using CopilotChat.WebApi.Storage; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; namespace CopilotChat.WebApi.Controllers; diff --git a/webapi/Controllers/DocumentController.cs b/webapi/Controllers/DocumentController.cs index 48c9fe565..f70ecf1bb 100644 --- a/webapi/Controllers/DocumentController.cs +++ b/webapi/Controllers/DocumentController.cs @@ -1,11 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Globalization; -using System.IO; -using System.Linq; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Hubs; @@ -15,10 +10,8 @@ using CopilotChat.WebApi.Options; using CopilotChat.WebApi.Services; using CopilotChat.WebApi.Storage; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; diff --git a/webapi/Controllers/MaintenanceController.cs b/webapi/Controllers/MaintenanceController.cs index 62b0f2ef5..551b3689e 100644 --- a/webapi/Controllers/MaintenanceController.cs +++ b/webapi/Controllers/MaintenanceController.cs @@ -1,11 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Threading; using CopilotChat.WebApi.Models.Response; using CopilotChat.WebApi.Options; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace CopilotChat.WebApi.Controllers; diff --git a/webapi/Controllers/PluginController.cs b/webapi/Controllers/PluginController.cs index be2ad8feb..f7d48a700 100644 --- a/webapi/Controllers/PluginController.cs +++ b/webapi/Controllers/PluginController.cs @@ -1,9 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Models.Storage; @@ -11,10 +7,8 @@ using CopilotChat.WebApi.Storage; using CopilotChat.WebApi.Utilities; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; namespace CopilotChat.WebApi.Controllers; diff --git a/webapi/Controllers/ServiceInfoController.cs b/webapi/Controllers/ServiceInfoController.cs index b7a2b961f..a00c785da 100644 --- a/webapi/Controllers/ServiceInfoController.cs +++ b/webapi/Controllers/ServiceInfoController.cs @@ -1,17 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Reflection; using CopilotChat.WebApi.Models.Response; using CopilotChat.WebApi.Options; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; diff --git a/webapi/Controllers/SpeechTokenController.cs b/webapi/Controllers/SpeechTokenController.cs index 787f8a572..2fb25fb84 100644 --- a/webapi/Controllers/SpeechTokenController.cs +++ b/webapi/Controllers/SpeechTokenController.cs @@ -1,13 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. using System.Net; -using System.Net.Http; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Response; using CopilotChat.WebApi.Options; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace CopilotChat.WebApi.Controllers; diff --git a/webapi/CopilotChatWebApi.csproj b/webapi/CopilotChatWebApi.csproj index 26c2539e6..7d3b24493 100644 --- a/webapi/CopilotChatWebApi.csproj +++ b/webapi/CopilotChatWebApi.csproj @@ -2,13 +2,7 @@ CopilotChat.WebApi net8.0 - LatestMajor - 10 - enable - disable 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 - All - SKEXP0003,SKEXP0011,SKEXP0021,SKEXP0026,SKEXP0042,SKEXP0050,SKEXP0052,SKEXP0053,SKEXP0060 @@ -19,60 +13,28 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - <_Parameter1>false diff --git a/webapi/Extensions/AsyncEnumerableExtensions.cs b/webapi/Extensions/AsyncEnumerableExtensions.cs index f50a4b593..c07a5359b 100644 --- a/webapi/Extensions/AsyncEnumerableExtensions.cs +++ b/webapi/Extensions/AsyncEnumerableExtensions.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Threading.Tasks; - namespace CopilotChat.WebApi.Extensions; /// diff --git a/webapi/Extensions/ConfigurationExtensions.cs b/webapi/Extensions/ConfigurationExtensions.cs index e89c1d75f..ace13d6ac 100644 --- a/webapi/Extensions/ConfigurationExtensions.cs +++ b/webapi/Extensions/ConfigurationExtensions.cs @@ -1,10 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Reflection; using Azure.Identity; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Hosting; namespace CopilotChat.WebApi.Extensions; diff --git a/webapi/Extensions/ExceptionExtensions.cs b/webapi/Extensions/ExceptionExtensions.cs index 4581dc753..7bd3d1563 100644 --- a/webapi/Extensions/ExceptionExtensions.cs +++ b/webapi/Extensions/ExceptionExtensions.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Threading; - #pragma warning disable IDE0130 // ReSharper disable once CheckNamespace - Using NS of Exception namespace System; diff --git a/webapi/Extensions/KernelMemoryClientExtensions.cs b/webapi/Extensions/KernelMemoryClientExtensions.cs index 77e147bb7..5c9a8de1d 100644 --- a/webapi/Extensions/KernelMemoryClientExtensions.cs +++ b/webapi/Extensions/KernelMemoryClientExtensions.cs @@ -1,18 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.Shared; +using CopilotChat.Shared.Ocr.Tesseract; using CopilotChat.WebApi.Models.Storage; using CopilotChat.WebApi.Services; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; +using Microsoft.KernelMemory.DataFormats.AzureAIDocIntel; namespace CopilotChat.WebApi.Extensions; @@ -50,7 +44,29 @@ public static void AddKernelMemoryServices(this WebApplicationBuilder appBuilder { if (hasOcr) { - memoryBuilder.WithCustomOcr(appBuilder.Configuration); + // Image OCR + switch (ocrType) + { + case string x when x.Equals("AzureAIDocIntel", StringComparison.OrdinalIgnoreCase): + { + AzureAIDocIntelConfig? cfg = appBuilder.Configuration + .GetSection($"{MemoryConfiguration.KernelMemorySection}:{MemoryConfiguration.ServicesSection}:AzureAIDocIntel") + .Get() ?? throw new ConfigurationException("Missing Azure AI Document Intelligence configuration"); + memoryBuilder.Services.AddSingleton(cfg); + memoryBuilder.WithCustomImageOcr(); + break; + } + + case string x when x.Equals("Tesseract", StringComparison.OrdinalIgnoreCase): + { + TesseractConfig? cfg = appBuilder.Configuration + .GetSection($"{MemoryConfiguration.KernelMemorySection}:{MemoryConfiguration.ServicesSection}:Tesseract") + .Get() ?? throw new ConfigurationException("Missing Tesseract configuration"); + memoryBuilder.Services.AddSingleton(cfg); + memoryBuilder.WithCustomImageOcr(); + break; + } + } } } diff --git a/webapi/Extensions/SemanticKernelExtensions.cs b/webapi/Extensions/SemanticKernelExtensions.cs index 219a42172..3b9247436 100644 --- a/webapi/Extensions/SemanticKernelExtensions.cs +++ b/webapi/Extensions/SemanticKernelExtensions.cs @@ -1,22 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.IO; -using System.Linq; -using System.Net.Http; using System.Reflection; -using System.Threading.Tasks; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Models.Response; using CopilotChat.WebApi.Options; using CopilotChat.WebApi.Plugins.Chat; using CopilotChat.WebApi.Services; using CopilotChat.WebApi.Storage; -using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; diff --git a/webapi/Extensions/ServiceExtensions.cs b/webapi/Extensions/ServiceExtensions.cs index 4b8d6e790..53f6c187f 100644 --- a/webapi/Extensions/ServiceExtensions.cs +++ b/webapi/Extensions/ServiceExtensions.cs @@ -1,9 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; using System.Reflection; using CopilotChat.Shared; using CopilotChat.WebApi.Auth; @@ -15,9 +11,6 @@ using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Identity.Web; using Microsoft.KernelMemory; diff --git a/webapi/Hubs/MessageRelayHub.cs b/webapi/Hubs/MessageRelayHub.cs index f01233dbd..a00e84bb6 100644 --- a/webapi/Hubs/MessageRelayHub.cs +++ b/webapi/Hubs/MessageRelayHub.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Threading.Tasks; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; namespace CopilotChat.WebApi.Hubs; diff --git a/webapi/Models/Request/Ask.cs b/webapi/Models/Request/Ask.cs index 3f7521bf3..157b88046 100644 --- a/webapi/Models/Request/Ask.cs +++ b/webapi/Models/Request/Ask.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; -using System.Linq; using CopilotChat.WebApi.Options; namespace CopilotChat.WebApi.Models.Request; diff --git a/webapi/Models/Request/DocumentImportForm.cs b/webapi/Models/Request/DocumentImportForm.cs index 65623b392..81bf43cdd 100644 --- a/webapi/Models/Request/DocumentImportForm.cs +++ b/webapi/Models/Request/DocumentImportForm.cs @@ -1,9 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Linq; -using Microsoft.AspNetCore.Http; - namespace CopilotChat.WebApi.Models.Request; /// diff --git a/webapi/Models/Request/DocumentStatusForm.cs b/webapi/Models/Request/DocumentStatusForm.cs index 8415a791f..6f200f2d5 100644 --- a/webapi/Models/Request/DocumentStatusForm.cs +++ b/webapi/Models/Request/DocumentStatusForm.cs @@ -1,9 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Linq; - namespace CopilotChat.WebApi.Models.Request; /// diff --git a/webapi/Models/Response/AskResult.cs b/webapi/Models/Response/AskResult.cs index e7a1a9d06..17b4c75c0 100644 --- a/webapi/Models/Response/AskResult.cs +++ b/webapi/Models/Response/AskResult.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Linq; - namespace CopilotChat.WebApi.Models.Response; public class AskResult diff --git a/webapi/Models/Response/ChatArchive.cs b/webapi/Models/Response/ChatArchive.cs index f37193324..5ed0020a2 100644 --- a/webapi/Models/Response/ChatArchive.cs +++ b/webapi/Models/Response/ChatArchive.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; using CopilotChat.WebApi.Models.Storage; using CopilotChat.WebApi.Options; using Microsoft.KernelMemory; diff --git a/webapi/Models/Response/DocumentMessageContent.cs b/webapi/Models/Response/DocumentMessageContent.cs index 1379551fb..999b1b3f6 100644 --- a/webapi/Models/Response/DocumentMessageContent.cs +++ b/webapi/Models/Response/DocumentMessageContent.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using CopilotChat.WebApi.Models.Request; diff --git a/webapi/Models/Response/ServiceInfoResponse.cs b/webapi/Models/Response/ServiceInfoResponse.cs index fb8f8fc47..526040c1e 100644 --- a/webapi/Models/Response/ServiceInfoResponse.cs +++ b/webapi/Models/Response/ServiceInfoResponse.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Linq; using System.Text.Json.Serialization; using CopilotChat.WebApi.Options; diff --git a/webapi/Models/Storage/ChatParticipant.cs b/webapi/Models/Storage/ChatParticipant.cs index 11bd86e05..a4ca6e486 100644 --- a/webapi/Models/Storage/ChatParticipant.cs +++ b/webapi/Models/Storage/ChatParticipant.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Text.Json.Serialization; using CopilotChat.WebApi.Storage; diff --git a/webapi/Models/Storage/ChatSession.cs b/webapi/Models/Storage/ChatSession.cs index c7a1cd6a6..05a976028 100644 --- a/webapi/Models/Storage/ChatSession.cs +++ b/webapi/Models/Storage/ChatSession.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Text.Json.Serialization; using CopilotChat.WebApi.Storage; diff --git a/webapi/Models/Storage/CopilotChatMessage.cs b/webapi/Models/Storage/CopilotChatMessage.cs index 16eb1351c..4cbe594fe 100644 --- a/webapi/Models/Storage/CopilotChatMessage.cs +++ b/webapi/Models/Storage/CopilotChatMessage.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; diff --git a/webapi/Models/Storage/MemorySource.cs b/webapi/Models/Storage/MemorySource.cs index a9488ba46..f550a316d 100644 --- a/webapi/Models/Storage/MemorySource.cs +++ b/webapi/Models/Storage/MemorySource.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Text.Json.Serialization; using CopilotChat.WebApi.Storage; diff --git a/webapi/Options/DocumentMemoryOptions.cs b/webapi/Options/DocumentMemoryOptions.cs index 5d50aa6b8..2cb6d920b 100644 --- a/webapi/Options/DocumentMemoryOptions.cs +++ b/webapi/Options/DocumentMemoryOptions.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.ComponentModel.DataAnnotations; namespace CopilotChat.WebApi.Options; diff --git a/webapi/Options/MemoryStoreType.cs b/webapi/Options/MemoryStoreType.cs index d68152336..b00f47a1e 100644 --- a/webapi/Options/MemoryStoreType.cs +++ b/webapi/Options/MemoryStoreType.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using Microsoft.Extensions.Configuration; using Microsoft.KernelMemory; using Microsoft.KernelMemory.MemoryStorage.DevTools; diff --git a/webapi/Options/NotEmptyOrWhitespaceAttribute.cs b/webapi/Options/NotEmptyOrWhitespaceAttribute.cs index a38dd2857..8350d14ab 100644 --- a/webapi/Options/NotEmptyOrWhitespaceAttribute.cs +++ b/webapi/Options/NotEmptyOrWhitespaceAttribute.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.ComponentModel.DataAnnotations; namespace CopilotChat.WebApi.Options; diff --git a/webapi/Options/PluginOptions.cs b/webapi/Options/PluginOptions.cs index cad852455..7f1ef4d4e 100644 --- a/webapi/Options/PluginOptions.cs +++ b/webapi/Options/PluginOptions.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; - namespace CopilotChat.WebApi.Options; /// diff --git a/webapi/Options/PromptsOptions.cs b/webapi/Options/PromptsOptions.cs index 0f8c9cf29..671a16128 100644 --- a/webapi/Options/PromptsOptions.cs +++ b/webapi/Options/PromptsOptions.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using CopilotChat.WebApi.Models.Request; diff --git a/webapi/Options/RequiredOnPropertyValueAttribute.cs b/webapi/Options/RequiredOnPropertyValueAttribute.cs index a46a395a7..65611fd91 100644 --- a/webapi/Options/RequiredOnPropertyValueAttribute.cs +++ b/webapi/Options/RequiredOnPropertyValueAttribute.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.ComponentModel.DataAnnotations; using System.Reflection; diff --git a/webapi/Plugins/Chat/ChatPlugin.cs b/webapi/Plugins/Chat/ChatPlugin.cs index 6b29afcb0..9b01f4b76 100644 --- a/webapi/Plugins/Chat/ChatPlugin.cs +++ b/webapi/Plugins/Chat/ChatPlugin.cs @@ -1,13 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.ComponentModel; using System.Globalization; -using System.Linq; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Auth; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Models.Response; @@ -17,7 +12,6 @@ using CopilotChat.WebApi.Services; using CopilotChat.WebApi.Storage; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; diff --git a/webapi/Plugins/Chat/KernelMemoryRetriever.cs b/webapi/Plugins/Chat/KernelMemoryRetriever.cs index bb087a8c5..35a74ce3d 100644 --- a/webapi/Plugins/Chat/KernelMemoryRetriever.cs +++ b/webapi/Plugins/Chat/KernelMemoryRetriever.cs @@ -1,17 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.ComponentModel; -using System.Linq; using System.Text; -using System.Threading.Tasks; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Models.Storage; using CopilotChat.WebApi.Options; using CopilotChat.WebApi.Plugins.Utils; using CopilotChat.WebApi.Storage; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; @@ -98,9 +93,9 @@ public KernelMemoryRetriever( FormatMemories(); FormatSnippets(); - /// - /// Format long term and working memories. - /// + // + // Format long term and working memories. + // void FormatMemories() { foreach (var memoryName in this._promptOptions.MemoryMap.Keys) @@ -121,9 +116,9 @@ void FormatMemories() } } - /// - /// Format document snippets. - /// + // + // Format document snippets. + // void FormatSnippets() { if (!memoryMap.TryGetValue(this._promptOptions.DocumentMemoryName, out var memories) || memories.Count == 0) @@ -145,9 +140,9 @@ void FormatSnippets() return (builderMemory.Length == 0 ? string.Empty : builderMemory.ToString(), citationMap); - /// - /// Search the memory for relevant memories by memory name. - /// + // + // Search the memory for relevant memories by memory name. + // async Task SearchMemoryAsync(string memoryName, bool isGlobalMemory = false) { var searchResult = @@ -164,10 +159,10 @@ await this._memoryClient.SearchMemoryAsync( } } - /// - /// Process the relevant memories and return a map of memories with citations for each memory name. - /// - /// A map of memories for each memory name and a map of citations for documents. + // + // Process the relevant memories and return a map of memories with citations for each memory name. + // + // A map of memories for each memory name and a map of citations for documents. (IDictionary>, IDictionary) ProcessMemories() { var memoryMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); diff --git a/webapi/Plugins/Chat/MsGraphOboPlugin.cs b/webapi/Plugins/Chat/MsGraphOboPlugin.cs index 4ce0592de..60069c96e 100644 --- a/webapi/Plugins/Chat/MsGraphOboPlugin.cs +++ b/webapi/Plugins/Chat/MsGraphOboPlugin.cs @@ -1,14 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.ComponentModel; -using System.Net.Http; +using System.Net.Http.Headers; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Options; -using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; namespace CopilotChat.WebApi.Plugins.Chat; @@ -107,7 +102,7 @@ public async Task CallGraphApiTasksAsync([Description("The URI of the Gr { using (var graphRequest = new HttpRequestMessage(HttpMethod.Get, apiToCall)) { - graphRequest.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", oboAccessToken); + graphRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", oboAccessToken); var graphResponse = await client.SendAsync(graphRequest, cancellationToken); if (graphResponse.IsSuccessStatusCode) diff --git a/webapi/Plugins/Chat/SemanticChatMemory.cs b/webapi/Plugins/Chat/SemanticChatMemory.cs index e01684553..86b3f1b96 100644 --- a/webapi/Plugins/Chat/SemanticChatMemory.cs +++ b/webapi/Plugins/Chat/SemanticChatMemory.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; diff --git a/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs b/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs index 11cb1c579..4a2cdfb20 100644 --- a/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs +++ b/webapi/Plugins/Chat/SemanticChatMemoryExtractor.cs @@ -1,15 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Globalization; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Models.Request; using CopilotChat.WebApi.Options; using CopilotChat.WebApi.Plugins.Utils; -using Microsoft.Extensions.Logging; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.OpenAI; @@ -64,9 +59,9 @@ public static async Task ExtractSemanticChatMemoryAsync( } } - /// - /// Extracts the semantic chat memory from the chat session. - /// + // + // Extracts the semantic chat memory from the chat session. + // async Task ExtractCognitiveMemoryAsync(string memoryType, string memoryName, ILogger logger) { if (!options.MemoryMap.TryGetValue(memoryName, out var memoryPrompt)) @@ -116,10 +111,10 @@ async Task ExtractCognitiveMemoryAsync(string memoryType, st return memory; } - /// - /// Create a memory item in the memory collection. - /// If there is already a memory item that has a high similarity score with the new item, it will be skipped. - /// + // + // Create a memory item in the memory collection. + // If there is already a memory item that has a high similarity score with the new item, it will be skipped. + // async Task CreateMemoryAsync(string memoryName, string memory) { try diff --git a/webapi/Plugins/OpenApi/GitHubPlugin/Model/PullRequest.cs b/webapi/Plugins/OpenApi/GitHubPlugin/Model/PullRequest.cs index d5d9ad266..39b399493 100644 --- a/webapi/Plugins/OpenApi/GitHubPlugin/Model/PullRequest.cs +++ b/webapi/Plugins/OpenApi/GitHubPlugin/Model/PullRequest.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Text.Json.Serialization; namespace CopilotChat.WebApi.Plugins.OpenApi.GitHubPlugin.Model; @@ -15,7 +13,7 @@ public class PullRequest /// Gets or sets the URL of the pull request /// [JsonPropertyName("url")] - public System.Uri Url { get; set; } + public Uri Url { get; set; } /// /// Gets or sets the unique identifier of the pull request @@ -92,7 +90,7 @@ public class PullRequest /// The date and time when the pull request was closed, or null if it is not closed. /// The date and time when the pull request was merged, or null if it is not merged. public PullRequest( - System.Uri url, + Uri url, int id, int number, string state, diff --git a/webapi/Plugins/OpenApi/GitHubPlugin/openapi.json b/webapi/Plugins/OpenApi/GitHubPlugin/openapi.json index e2cacc478..c2505d1dd 100644 --- a/webapi/Plugins/OpenApi/GitHubPlugin/openapi.json +++ b/webapi/Plugins/OpenApi/GitHubPlugin/openapi.json @@ -70,7 +70,7 @@ "/repos/{owner}/{repo}/pulls": { "get": { "summary": "List pull requests", - "description": "Draft pull requests are available in public repositories with GitHub Free and GitHub Free for organizations, GitHub Pro, and legacy per-repository billing plans, and in public and private repositories with GitHub Team and GitHub Enterprise Cloud. For more information, see [GitHub's products](https://docs.github.com/enterprise-cloud@latest//github/getting-started-with-github/githubs-products) in the GitHub Help documentation.", + "description": "", "tags": [ "pulls" ], @@ -197,7 +197,7 @@ "/repos/{owner}/{repo}/pulls/{pull_number}": { "get": { "summary": "Get a pull request", - "description": "Draft pull requests are available in public repositories with GitHub Free and GitHub Free for organizations, GitHub Pro, and legacy per-repository billing plans, and in public and private repositories with GitHub Team and GitHub Enterprise Cloud. For more information, see [GitHub's products](https://docs.github.com/enterprise-cloud@latest//github/getting-started-with-github/githubs-products) in the GitHub Help documentation.\n\nLists details of a pull request by providing its number.\n\nWhen you get, [create](https://docs.github.com/enterprise-cloud@latest//rest/reference/pulls/#create-a-pull-request), or [edit](https://docs.github.com/enterprise-cloud@latest//rest/reference/pulls#update-a-pull-request) a pull request, GitHub Enterprise Cloud creates a merge commit to test whether the pull request can be automatically merged into the base branch. This test commit is not added to the base branch or the head branch. You can review the status of the test commit using the `mergeable` key. For more information, see \"[Checking mergeability of pull requests](https://docs.github.com/enterprise-cloud@latest//rest/guides/getting-started-with-the-git-database-api#checking-mergeability-of-pull-requests)\".\n\nThe value of the `mergeable` attribute can be `true`, `false`, or `null`. If the value is `null`, then GitHub Enterprise Cloud has started a background job to compute the mergeability. After giving the job time to complete, resubmit the request. When the job finishes, you will see a non-`null` value for the `mergeable` attribute in the response. If `mergeable` is `true`, then `merge_commit_sha` will be the SHA of the _test_ merge commit.\n\nThe value of the `merge_commit_sha` attribute changes depending on the state of the pull request. Before merging a pull request, the `merge_commit_sha` attribute holds the SHA of the _test_ merge commit. After merging a pull request, the `merge_commit_sha` attribute changes depending on how you merged the pull request:\n\n* If merged as a [merge commit](https://docs.github.com/enterprise-cloud@latest//articles/about-merge-methods-on-github/), `merge_commit_sha` represents the SHA of the merge commit.\n* If merged via a [squash](https://docs.github.com/enterprise-cloud@latest//articles/about-merge-methods-on-github/#squashing-your-merge-commits), `merge_commit_sha` represents the SHA of the squashed commit on the base branch.\n* If [rebased](https://docs.github.com/enterprise-cloud@latest//articles/about-merge-methods-on-github/#rebasing-and-merging-your-commits), `merge_commit_sha` represents the commit that the base branch was updated to.\n\nPass the appropriate [media type](https://docs.github.com/enterprise-cloud@latest//rest/overview/media-types/#commits-commit-comparison-and-pull-requests) to fetch diff and patch formats.", + "description": "Lists details of a pull request by providing its number.", "tags": [ "pulls" ], @@ -219,7 +219,7 @@ ], "responses": { "200": { - "description": "Pass the appropriate [media type](https://docs.github.com/enterprise-cloud@latest//rest/overview/media-types/#commits-commit-comparison-and-pull-requests) to fetch diff and patch formats.", + "description": "Pass the appropriate [media type] to fetch diff and patch formats.", "content": { "application/json": { "schema": { diff --git a/webapi/Plugins/OpenApi/JiraPlugin/Model/CommentResponse.cs b/webapi/Plugins/OpenApi/JiraPlugin/Model/CommentResponse.cs index 1058046de..36be3129e 100644 --- a/webapi/Plugins/OpenApi/JiraPlugin/Model/CommentResponse.cs +++ b/webapi/Plugins/OpenApi/JiraPlugin/Model/CommentResponse.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; using System.Text.Json.Serialization; namespace CopilotChat.WebApi.Plugins.OpenApi.JiraPlugin.Model; diff --git a/webapi/Plugins/Utils/AsyncUtils.cs b/webapi/Plugins/Utils/AsyncUtils.cs index 1bb48f855..6da795e05 100644 --- a/webapi/Plugins/Utils/AsyncUtils.cs +++ b/webapi/Plugins/Utils/AsyncUtils.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Threading.Tasks; using Microsoft.SemanticKernel; namespace CopilotChat.WebApi.Plugins.Utils; diff --git a/webapi/Plugins/Utils/JsonUtils.cs b/webapi/Plugins/Utils/JsonUtils.cs index 5f7f668aa..203c85d42 100644 --- a/webapi/Plugins/Utils/JsonUtils.cs +++ b/webapi/Plugins/Utils/JsonUtils.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using System.Globalization; using System.Text.Json; using System.Text.RegularExpressions; @@ -58,7 +56,7 @@ internal static string OptimizeOdataResponseJson(string jsonString, int tokenLim resultsDescriptor = string.Format(CultureInfo.InvariantCulture, "{0}: ", odataContextVal); } - // Extract object to be truncated + // Extract object to be truncated var valueDocument = JsonDocument.Parse(valueElement.GetRawText()); document = valueDocument; } diff --git a/webapi/Plugins/Utils/TokenUtils.cs b/webapi/Plugins/Utils/TokenUtils.cs index 355af59fb..09263a7a8 100644 --- a/webapi/Plugins/Utils/TokenUtils.cs +++ b/webapi/Plugins/Utils/TokenUtils.cs @@ -1,12 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; using System.Globalization; -using System.Linq; using System.Text.Json; -using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; +using SharpToken; namespace CopilotChat.WebApi.Plugins.Utils; @@ -15,7 +13,7 @@ namespace CopilotChat.WebApi.Plugins.Utils; /// public static class TokenUtils { - private static readonly SharpToken.GptEncoding s_tokenizer = SharpToken.GptEncoding.GetEncoding("cl100k_base"); + private static readonly GptEncoding s_tokenizer = GptEncoding.GetEncoding("cl100k_base"); /// /// Semantic dependencies of ChatPlugin. diff --git a/webapi/Program.cs b/webapi/Program.cs index c4dd756eb..57bced375 100644 --- a/webapi/Program.cs +++ b/webapi/Program.cs @@ -1,22 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Diagnostics; -using System.Linq; using System.Text.Json; -using System.Threading.Tasks; using CopilotChat.WebApi.Extensions; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Services; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.Extensibility.Implementation; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; namespace CopilotChat.WebApi; diff --git a/webapi/Services/AppInsightsTelemetryService.cs b/webapi/Services/AppInsightsTelemetryService.cs index 9f38d2a41..5e0dafdb2 100644 --- a/webapi/Services/AppInsightsTelemetryService.cs +++ b/webapi/Services/AppInsightsTelemetryService.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; using System.Security.Claims; using Microsoft.ApplicationInsights; -using Microsoft.AspNetCore.Http; namespace CopilotChat.WebApi.Services; diff --git a/webapi/Services/AppInsightsUserTelemetryInitializerService.cs b/webapi/Services/AppInsightsUserTelemetryInitializerService.cs index c0097f4d0..dc4a029a1 100644 --- a/webapi/Services/AppInsightsUserTelemetryInitializerService.cs +++ b/webapi/Services/AppInsightsUserTelemetryInitializerService.cs @@ -3,7 +3,6 @@ using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.DataContracts; using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.AspNetCore.Http; namespace CopilotChat.WebApi.Services; diff --git a/webapi/Services/AzureContentSafety.cs b/webapi/Services/AzureContentSafety.cs index f1c98198d..7eae57b51 100644 --- a/webapi/Services/AzureContentSafety.cs +++ b/webapi/Services/AzureContentSafety.cs @@ -1,16 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Response; -using Microsoft.AspNetCore.Http; using Microsoft.SemanticKernel; // ReSharper disable MissingLinebreak diff --git a/webapi/Services/DocumentTypeProvider.cs b/webapi/Services/DocumentTypeProvider.cs index 66d2e7c9c..19ee7c58b 100644 --- a/webapi/Services/DocumentTypeProvider.cs +++ b/webapi/Services/DocumentTypeProvider.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; using Microsoft.KernelMemory.Pipeline; namespace CopilotChat.WebApi.Services; diff --git a/webapi/Services/IContentSafetyService.cs b/webapi/Services/IContentSafetyService.cs index 4f9420b1d..1adb7cbb2 100644 --- a/webapi/Services/IContentSafetyService.cs +++ b/webapi/Services/IContentSafetyService.cs @@ -1,11 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Response; -using Microsoft.AspNetCore.Http; namespace CopilotChat.WebApi.Services; diff --git a/webapi/Services/IMaintenanceAction.cs b/webapi/Services/IMaintenanceAction.cs index a4848c85c..dbcd20842 100644 --- a/webapi/Services/IMaintenanceAction.cs +++ b/webapi/Services/IMaintenanceAction.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Threading; -using System.Threading.Tasks; - namespace CopilotChat.WebApi.Services; /// diff --git a/webapi/Services/MaintenanceMiddleware.cs b/webapi/Services/MaintenanceMiddleware.cs index 0edd5165e..a0f406026 100644 --- a/webapi/Services/MaintenanceMiddleware.cs +++ b/webapi/Services/MaintenanceMiddleware.cs @@ -1,13 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Threading.Tasks; using CopilotChat.WebApi.Controllers; using CopilotChat.WebApi.Hubs; using CopilotChat.WebApi.Options; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.SemanticKernel; diff --git a/webapi/Services/SemanticKernelProvider.cs b/webapi/Services/SemanticKernelProvider.cs index 796e8609d..86f084d14 100644 --- a/webapi/Services/SemanticKernelProvider.cs +++ b/webapi/Services/SemanticKernelProvider.cs @@ -1,9 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Net.Http; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using Microsoft.KernelMemory; using Microsoft.SemanticKernel; diff --git a/webapi/Storage/ChatMemorySourceRepository.cs b/webapi/Storage/ChatMemorySourceRepository.cs index 427a701e6..b02e86c0a 100644 --- a/webapi/Storage/ChatMemorySourceRepository.cs +++ b/webapi/Storage/ChatMemorySourceRepository.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/ChatMessageRepository.cs b/webapi/Storage/ChatMessageRepository.cs index 2e539ce9e..a4fac0f00 100644 --- a/webapi/Storage/ChatMessageRepository.cs +++ b/webapi/Storage/ChatMessageRepository.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/ChatParticipantRepository.cs b/webapi/Storage/ChatParticipantRepository.cs index 313041b76..37d9efc63 100644 --- a/webapi/Storage/ChatParticipantRepository.cs +++ b/webapi/Storage/ChatParticipantRepository.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/ChatSessionRepository.cs b/webapi/Storage/ChatSessionRepository.cs index 726b50d2e..34454c27a 100644 --- a/webapi/Storage/ChatSessionRepository.cs +++ b/webapi/Storage/ChatSessionRepository.cs @@ -1,7 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/CosmosDbContext.cs b/webapi/Storage/CosmosDbContext.cs index dbbc45944..1a6440fa6 100644 --- a/webapi/Storage/CosmosDbContext.cs +++ b/webapi/Storage/CosmosDbContext.cs @@ -1,10 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; using Microsoft.Azure.Cosmos; diff --git a/webapi/Storage/FileSystemContext.cs b/webapi/Storage/FileSystemContext.cs index de6b21f6f..6ec77f54b 100644 --- a/webapi/Storage/FileSystemContext.cs +++ b/webapi/Storage/FileSystemContext.cs @@ -1,12 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Text.Json; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/IRepository.cs b/webapi/Storage/IRepository.cs index 104f65bc9..398724b34 100644 --- a/webapi/Storage/IRepository.cs +++ b/webapi/Storage/IRepository.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Threading.Tasks; - namespace CopilotChat.WebApi.Storage; /// diff --git a/webapi/Storage/IStorageContext.cs b/webapi/Storage/IStorageContext.cs index 4494b979e..17b63fb3e 100644 --- a/webapi/Storage/IStorageContext.cs +++ b/webapi/Storage/IStorageContext.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/Repository.cs b/webapi/Storage/Repository.cs index f22fb6eb0..4e5e00296 100644 --- a/webapi/Storage/Repository.cs +++ b/webapi/Storage/Repository.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Storage/VolatileContext.cs b/webapi/Storage/VolatileContext.cs index 10d6c834b..0374cc407 100644 --- a/webapi/Storage/VolatileContext.cs +++ b/webapi/Storage/VolatileContext.cs @@ -1,11 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Threading.Tasks; using CopilotChat.WebApi.Models.Storage; namespace CopilotChat.WebApi.Storage; diff --git a/webapi/Utilities/PluginUtils.cs b/webapi/Utilities/PluginUtils.cs index 58ac661c3..cb7cfc881 100644 --- a/webapi/Utilities/PluginUtils.cs +++ b/webapi/Utilities/PluginUtils.cs @@ -1,8 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using CopilotChat.WebApi.Models.Request; - namespace CopilotChat.WebApi.Utilities; /// @@ -28,10 +25,12 @@ public static Uri GetPluginManifestUri(string manifestDomain) /// The plugin manifest URI. public static Uri GetPluginManifestUri(Uri manifestDomain) { - UriBuilder uriBuilder = new(manifestDomain); + UriBuilder uriBuilder = new(manifestDomain) + { + // Expected manifest path as defined by OpenAI: https://platform.openai.com/docs/plugins/getting-started/plugin-manifest + Path = "/.well-known/ai-plugin.json" + }; - // Expected manifest path as defined by OpenAI: https://platform.openai.com/docs/plugins/getting-started/plugin-manifest - uriBuilder.Path = "/.well-known/ai-plugin.json"; return uriBuilder.Uri; } diff --git a/webapi/appsettings.json b/webapi/appsettings.json index a62a31a32..86743a722 100644 --- a/webapi/appsettings.json +++ b/webapi/appsettings.json @@ -46,12 +46,12 @@ // - ManifestDomain is the root domain of the plugin: https://platform.openai.com/docs/plugins/production/domain-verification-and-security // - Key is the key used to access the plugin if it requires authentication. "Plugins": [ - // Klarna Shopping does not require authentication. - { - "Name": "Klarna Shopping", - "ManifestDomain": "https://www.klarna.com" - // "Key": "" - } +// // Klarna Shopping does not require authentication. +// { +// "Name": "Klarna Shopping", +// "ManifestDomain": "https://www.klarna.com" +// // "Key": "" +// } ], // // Optional Azure Speech service configuration for providing Azure Speech access tokens. @@ -208,6 +208,7 @@ // - Directory is the location where files are stored. // "SimpleFileStorage": { + "StorageType": "Disk", "Directory": "../tmp/cache" }, // @@ -227,6 +228,29 @@ "Directory": "../tmp/database" }, // + // Azure Cognitive Search configuration for semantic services. + // - Auth is the authentication type: "APIKey" or "AzureIdentity". + // - APIKey is the key generated to access the service. + // - Endpoint is the service endpoint url. + // - UseHybridSearch is whether to use also text search, disabled by default + // + "AzureAISearch": { + "Auth": "ApiKey", + //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureAISearch:APIKey" "MY_ACS_KEY" + "Endpoint": "" + }, + // + // Azure Form Recognizer configuration for memory pipeline OCR. + // - Auth is the authentication configuration: "APIKey" or "AzureIdentity". + // - APIKey is the key generated to access the service. + // - Endpoint is the service endpoint url. + // + "AzureAIDocIntel": { + "Auth": "APIKey", + //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureAIDocIntel:APIKey" "MY_AZURE_AI_DOC_INTEL_KEY" + "Endpoint": "" + }, + // // Azure blob storage for the memory pipeline // - Auth is the authentication type: "ConnectionString" or "AzureIdentity". // - ConnectionString is the connection string for the Azure Storage account and only utilized when Auth=ConnectionString. @@ -242,50 +266,20 @@ //"EndpointSuffix": "core.windows.net" }, // - // Azure storage queue configuration for distributed memory pipeline - // - Auth is the authentication type: "ConnectionString" or "AzureIdentity". - // - ConnectionString is the connection string for the Azure Storage account and only utilized when Auth=ConnectionString. - // - Account is the name of the Azure Storage account and only utilized when Auth=AzureIdentity. - // - EndpointSuffix is used only for country clouds. - // - "AzureQueue": { - "Auth": "ConnectionString" - //"ConnectionString": "", // dotnet user-secrets set "KernelMemory:Services:AzureQueue:ConnectionString" "MY_AZUREQUEUE_CONNECTIONSTRING" - //"Account": "", - //"EndpointSuffix": "core.windows.net" - }, - // - // RabbitMq queue configuration for distributed memory pipeline - // - Username is the RabbitMq user name. - // - Password is the RabbitMq use password - // - Host is the RabbitMq service host name or address. - // - Port is the RabbitMq service port. - // - "RabbitMq": { - //"Username": "user", // dotnet user-secrets set "KernelMemory:Services:RabbitMq:Username" "MY_RABBITMQ_USER" - //"Password": "", // dotnet user-secrets set "KernelMemory:Services:RabbitMq:Password" "MY_RABBITMQ_KEY" - "Host": "127.0.0.1", - "Port": "5672" - }, - // - // Azure Cognitive Search configuration for semantic services. + // AI embedding configuration for Azure OpenAI services. // - Auth is the authentication type: "APIKey" or "AzureIdentity". // - APIKey is the key generated to access the service. // - Endpoint is the service endpoint url. - // - "AzureAISearch": { + // - Deployment is an embedding model (e.g., text-embedding-ada-002). + // - MaxTokenTotal defaults to 8191 + // - EmbeddingDimensions is null or a number of dimensions to truncate embeddings + // - MaxEmbeddingBatchSize is by default 1 + // - MaxRetries is the number of times to retry generation in case of errors + "AzureOpenAIEmbedding": { "Auth": "ApiKey", - //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureAISearch:APIKey" "MY_ACS_KEY" - "Endpoint": "" - }, - // - // Qdrant configuration for semantic services. - // - APIKey is the key generated to access the service. - // - Endpoint is the service endpoint url. - // - "Qdrant": { - //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:Qdrant:APIKey" "MY_QDRANT_KEY" - "Endpoint": "http://127.0.0.1:6333" + // "APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureOpenAIEmbedding:APIKey" "MY_AZUREOPENAI_KEY" + "Endpoint": "", + "Deployment": "text-embedding-ada-002" }, // // AI completion configuration for Azure AI services. @@ -301,21 +295,140 @@ // "APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureOpenAIText:APIKey" "MY_AZUREOPENAI_KEY" "Endpoint": "", "Deployment": "gpt-4o", + "MaxTokenTotal": 16384, "APIType": "ChatCompletion", "MaxRetries": 10 }, // - // AI embedding configuration for Azure OpenAI services. - // - Auth is the authentication type: "APIKey" or "AzureIdentity". - // - APIKey is the key generated to access the service. - // - Endpoint is the service endpoint url. - // - Deployment is an embedding model (e.g., text-embedding-ada-002). + // Azure storage queue configuration for distributed memory pipeline + // - Auth is the authentication type: "ConnectionString" or "AzureIdentity". + // - ConnectionString is the connection string for the Azure Storage account and only utilized when Auth=ConnectionString. + // - Account is the name of the Azure Storage account and only utilized when Auth=AzureIdentity. + // - EndpointSuffix is used only for country clouds. // - "AzureOpenAIEmbedding": { - "Auth": "ApiKey", - // "APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureOpenAIEmbedding:APIKey" "MY_AZUREOPENAI_KEY" - "Endpoint": "", - "Deployment": "text-embedding-ada-002" + "AzureQueue": { + "Auth": "ConnectionString", + //"ConnectionString": "", // dotnet user-secrets set "KernelMemory:Services:AzureQueue:ConnectionString" "MY_AZUREQUEUE_CONNECTIONSTRING" + //"Account": "", + //"EndpointSuffix": "core.windows.net" + "PollDelayMsecs": 100, + "FetchBatchSize": 3, + "FetchLockSeconds": 300, + "MaxRetriesBeforePoisonQueue": 20, + "PoisonQueueSuffix": "-poison" + }, + "Ollama": { + "Endpoint": "http://localhost:11434", + "TextModel": { + "ModelName": "phi3:medium-128k", + "MaxTokenTotal": 131072, + // How many requests can be processed in parallel + "MaxBatchSize": 1 + //// Enable Mirostat sampling for controlling perplexity. + //// (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) + //"MiroStat": 0, + //// Influences how quickly the algorithm responds to feedback from the + //// generated text. A lower learning rate will result in slower adjustments, + //// while a higher learning rate will make the algorithm more responsive. + //// (Default: 0.1) + //"MiroStatEta": 0.1, + //// Controls the balance between coherence and diversity of the output. + //// A lower value will result in more focused and coherent text. + //// (Default: 5.0) + //"MiroStatTau": 5.0, + //// Sets the size of the context window used to generate the next token. + //// (Default: 2048) + //"NumCtx": 2048, + //// The number of GQA groups in the transformer layer. Required for some + //// models, for example it is 8 for llama2:70b + //"NumGqa": null, + //// The number of layers to send to the GPU(s). On macOS it defaults to + //// 1 to enable metal support, 0 to disable. + //"NumGpu": null, + //// Sets the number of threads to use during computation. By default, + //// Ollama will detect this for optimal performance. + //// It is recommended to set this value to the number of physical CPU cores + //// your system has (as opposed to the logical number of cores). + //"NumThread": null, + //// Sets how far back for the model to look back to prevent repetition. + //// (Default: 64, 0 = disabled, -1 = num_ctx) + //"RepeatLastN": null, + //// Sets the random number seed to use for generation. + //// Setting this to a specific number will make the model generate the same + //// text for the same prompt. (Default: 0) + //"Seed": 0, + //// Tail free sampling is used to reduce the impact of less probable + //// tokens from the output. A higher value (e.g., 2.0) will reduce the + //// impact more, while a value of 1.0 disables this setting. (default: 1) + //"TfsZ": 1.0, + //// Maximum number of tokens to predict when generating text. + //// (Default: 128, -1 = infinite generation, -2 = fill context) + //"NumPredict": 128, + //// Reduces the probability of generating nonsense. A higher value + //// (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) + //// will be more conservative. (Default: 40) + //"TopK": 40, + //// Alternative to the top_p, and aims to ensure a balance of quality and variety.min_p represents the minimum + //// probability for a token to be considered, relative to the probability of the most likely token.For + //// example, with min_p=0.05 and the most likely token having a probability of 0.9, logits with a value less + //// than 0.05*0.9=0.045 are filtered out. (Default: 0.0) + //"MinP": 0.0 + }, + "EmbeddingModel": { + "ModelName": "nomic-embed-text", + "MaxTokenTotal": 2048, + // How many requests can be processed in parallel + "MaxBatchSize": 1 + //// Enable Mirostat sampling for controlling perplexity. + //// (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) + //"MiroStat": 0, + //// Influences how quickly the algorithm responds to feedback from the + //// generated text. A lower learning rate will result in slower adjustments, + //// while a higher learning rate will make the algorithm more responsive. + //// (Default: 0.1) + //"MiroStatEta": 0.1, + //// Controls the balance between coherence and diversity of the output. + //// A lower value will result in more focused and coherent text. + //// (Default: 5.0) + //"MiroStatTau": 5.0, + //// Sets the size of the context window used to generate the next token. + //// (Default: 2048) + //"NumCtx": 2048, + //// The number of GQA groups in the transformer layer. Required for some + //// models, for example it is 8 for llama2:70b + //"NumGqa": null, + //// The number of layers to send to the GPU(s). On macOS it defaults to + //// 1 to enable metal support, 0 to disable. + //"NumGpu": null, + //// Sets the number of threads to use during computation. By default, + //// Ollama will detect this for optimal performance. + //// It is recommended to set this value to the number of physical CPU cores + //// your system has (as opposed to the logical number of cores). + //"NumThread": null, + //// Sets how far back for the model to look back to prevent repetition. + //// (Default: 64, 0 = disabled, -1 = num_ctx) + //"RepeatLastN": null, + //// Sets the random number seed to use for generation. + //// Setting this to a specific number will make the model generate the same + //// text for the same prompt. (Default: 0) + //"Seed": 0, + //// Tail free sampling is used to reduce the impact of less probable + //// tokens from the output. A higher value (e.g., 2.0) will reduce the + //// impact more, while a value of 1.0 disables this setting. (default: 1) + //"TfsZ": 1.0, + //// Maximum number of tokens to predict when generating text. + //// (Default: 128, -1 = infinite generation, -2 = fill context) + //"NumPredict": 128, + //// Reduces the probability of generating nonsense. A higher value + //// (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) + //// will be more conservative. (Default: 40) + //"TopK": 40, + //// Alternative to the top_p, and aims to ensure a balance of quality and variety.min_p represents the minimum + //// probability for a token to be considered, relative to the probability of the most likely token.For + //// example, with min_p=0.05 and the most likely token having a probability of 0.9, logits with a value less + //// than 0.05*0.9=0.045 are filtered out. (Default: 0.0) + //"MinP": 0.0 + } }, // // AI completion and embedding configuration for OpenAI services. @@ -328,26 +441,46 @@ "OpenAI": { "TextModel": "gpt-4o", "EmbeddingModel": "text-embedding-ada-002", + "EmbeddingModelMaxTokenTotal": 8191, //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:OpenAI:APIKey" "MY_OPENAI_KEY" "OrgId": "", - "MaxRetries": 10 + "MaxRetries": 10, + "MaxEmbeddingBatchSize": 100 + }, + "Postgres": { + // Postgres instance connection string + "ConnectionString": "Host=localhost;Port=5432;Username=public;Password=;Database=public", // dotnet user-secrets set "KernelMemory:Services:Postgres:ConnectionString" "MY POSTGRES CONNECTION STRING" + // Mandatory prefix to add to the name of table managed by KM, + // e.g. to exclude other tables in the same schema. + "TableNamePrefix": "km-" }, // - // Azure Form Recognizer configuration for memory pipeline OCR. - // - Auth is the authentication configuration: "APIKey" or "AzureIdentity". + // Qdrant configuration for semantic services. // - APIKey is the key generated to access the service. // - Endpoint is the service endpoint url. // - "AzureAIDocIntel": { - "Auth": "APIKey", - //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:AzureAIDocIntel:APIKey" "MY_AZURE_AI_DOC_INTEL_KEY" - "Endpoint": "" + "Qdrant": { + //"APIKey": "", // dotnet user-secrets set "KernelMemory:Services:Qdrant:APIKey" "MY_QDRANT_KEY" + "Endpoint": "http://127.0.0.1:6333" + }, + // + // RabbitMq queue configuration for distributed memory pipeline + // - Username is the RabbitMq user name. + // - Password is the RabbitMq use password + // - Host is the RabbitMq service host name or address. + // - Port is the RabbitMq service port. + // + "RabbitMQ": { + //"Username": "user", // dotnet user-secrets set "KernelMemory:Services:RabbitMq:Username" "MY_RABBITMQ_USER" + //"Password": "", // dotnet user-secrets set "KernelMemory:Services:RabbitMq:Password" "MY_RABBITMQ_KEY" + "Host": "127.0.0.1", + "Port": "5672" }, // // Tesseract configuration for memory pipeline OCR. // - Language is the language supported by the data file. // - FilePath is the path to the data file. - // + // // Note: When using Tesseract OCR Support (In order to upload image file formats such as png, jpg and tiff): // 1. Obtain language data files here: https://github.com/tesseract-ocr/tessdata . // 2. Add these files to your `data` folder or the path specified in the "FilePath" property and set the "Copy to Output Directory" value to "Copy if newer".