Misc upgrades (#1213)

* Upgrade Semantic Kernel * Upgrade Kernel Memory * Fix Github plugin * Handle OpenAI plugins deprecation * Cleanup code, refactoring, code style
microsoft · Nov 13, 2024 · 23b2782 · 23b2782
1 parent 046c539
commit 23b2782
Show file tree

Hide file tree

Showing 95 changed files with 618 additions and 736 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,8 @@ dotnet/.config
 *.user
 *.userosscache
 *.sln.docstates
+webapi/data/*.json
+__dev/
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs

diff --git a/CopilotChat.sln.DotSettings b/CopilotChat.sln.DotSettings
@@ -289,6 +289,7 @@ public void It$SOMENAME$()
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=subdir/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=SVCS/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=syntaxes/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=tesseract/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=testsettings/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=tiktoken/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=tldr/@EntryIndexedValue">True</s:Boolean>

diff --git a/Directory.Build.props b/Directory.Build.props
@@ -0,0 +1,51 @@
+<Project>
+  <PropertyGroup>
+    <!-- C# lang version, https://learn.microsoft.com/dotnet/csharp/whats-new -->
+    <LangVersion>12</LangVersion>
+
+    <!-- https://learn.microsoft.com/dotnet/core/tools/global-json#rollforward -->
+    <RollForward>LatestMajor</RollForward>
+
+    <!-- https://learn.microsoft.com/dotnet/csharp/language-reference/builtin-types/nullable-value-types -->
+    <Nullable>enable</Nullable>
+
+    <!-- https://devblogs.microsoft.com/dotnet/welcome-to-csharp-10 -->
+    <ImplicitUsings>enable</ImplicitUsings>
+
+    <IsPackable>false</IsPackable>
+    <AnalysisMode>All</AnalysisMode>
+    <NoWarn>CA1812,CA2234,CS1570,CS1572,CS1573,CS1574,CA1056,CA1716,CA1724,SKEXP0003,SKEXP0011,SKEXP0021,SKEXP0026,SKEXP0042,SKEXP0050,SKEXP0052,SKEXP0053,SKEXP0060,KMEXP02,SKEXP0040</NoWarn>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CodeAnalysis.Analyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.CodeStyle">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Roslynator.Analyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Roslynator.CodeAnalysis.Analyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Roslynator.Formatting.Analyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+</Project>
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -0,0 +1,102 @@
+<Project>
+  <PropertyGroup>
+    <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.2" />
+    <PackageVersion Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
+    <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.45.1" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.Identity.Client" Version="4.66.2" />
+    <PackageVersion Include="Microsoft.Identity.Web" Version="2.21.1" />
+    <PackageVersion Include="SharpToken" Version="2.0.3" />
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.9.0" />
+    <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+    <PackageVersion Include="Tesseract" Version="5.2.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <!-- Make sure the app is thoroughly tested after any Microsoft.Azure.Functions.* package updates. -->
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="1.23.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.2.0" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.OpenApi" Version="1.5.1" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.18.1" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.28.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.28.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" Version="1.28.0-preview" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.28.0-preview" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.SemanticKernel.Plugins.Core" Version="1.28.0-alpha" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.28.0-alpha" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Plugins.MsGraph" Version="1.28.0-alpha" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Plugins.OpenApi" Version="1.28.0-alpha" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Plugins.Web" Version="1.28.0-alpha" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.KernelMemory.AI.AzureOpenAI" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.AI.Ollama" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.AI.OpenAI" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.DataFormats.AzureAIDocIntel" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.DocumentStorage.AzureBlobs" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.MemoryDb.AzureAISearch" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.MemoryDb.Postgres" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.MemoryDb.SQLServer" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Orchestration.AzureQueues" Version="0.92.241112.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Orchestration.RabbitMQ" Version="0.92.241112.1" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageVersion Include="xunit" Version="2.9.2" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageVersion>
+    <PackageVersion Include="coverlet.collector" Version="6.0.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageVersion>
+  </ItemGroup>
+  <!-- Code Analysis -->
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.11.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Roslynator.Analyzers" Version="4.12.9">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Roslynator.CodeAnalysis.Analyzers" Version="4.12.9">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Roslynator.Formatting.Analyzers" Version="4.12.9">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.12.19">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+  </ItemGroup>
+</Project>
diff --git a/integration-tests/ChatCopilotIntegrationTest.cs b/integration-tests/ChatCopilotIntegrationTest.cs
@@ -1,10 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System;
-using System.Linq;
-using System.Net.Http;
 using System.Net.Http.Headers;
-using System.Threading.Tasks;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Identity.Client;
 using Xunit;
@@ -15,6 +11,7 @@ namespace ChatCopilotIntegrationTests;
 /// Base class for Chat Copilot integration tests
 /// </summary>
 [Trait("Category", "Integration Tests")]
+#pragma warning disable CA1051
 public abstract class ChatCopilotIntegrationTest : IDisposable
 {
     protected const string BaseUrlSettingName = "BaseServerUrl";
@@ -76,7 +73,7 @@ protected async Task<string> GetUserTokenByPasswordAsync()
         string? scopeString = this.Configuration[ScopesSettingName];
         Assert.NotNull(scopeString);
 
-        string[] scopes = scopeString.Split(new char[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries);
+        string[] scopes = scopeString.Split([',', ' '], StringSplitOptions.RemoveEmptyEntries);
 
         var accounts = await app.GetAccountsAsync();
 

diff --git a/integration-tests/ChatCopilotIntegrationTests.csproj b/integration-tests/ChatCopilotIntegrationTests.csproj
@@ -2,50 +2,35 @@
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
     <UserSecretsId>81136671-a63f-4f20-bd0e-a65b2561999a</UserSecretsId>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
-    <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.11.20">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
-    <PackageReference Include="Roslynator.Analyzers" Version="4.12.9">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
-    <PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.12.9">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
-    <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.12.9">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
-    <PackageReference Include="xunit" Version="2.9.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
+    <ProjectReference Include="..\webapi\CopilotChatWebApi.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="6.0.2">
+    <PackageReference Include="coverlet.collector">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\webapi\CopilotChatWebApi.csproj" />
-  </ItemGroup>
-
   <ItemGroup>
     <None Update="testsettings.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

diff --git a/integration-tests/ChatTests.cs b/integration-tests/ChatTests.cs
@@ -1,7 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System.Collections.Generic;
-using System.Net.Http;
 using System.Net.Http.Json;
 using System.Text.Json;
 using CopilotChat.WebApi.Models.Request;
@@ -13,6 +11,8 @@ namespace ChatCopilotIntegrationTests;
 
 public class ChatTests : ChatCopilotIntegrationTest
 {
+    private static readonly JsonSerializerOptions jsOpts = new() { PropertyNameCaseInsensitive = true };
+
     [Fact]
     public async void ChatMessagePostSucceedsWithValidInput()
     {
@@ -24,7 +24,7 @@ public async void ChatMessagePostSucceedsWithValidInput()
         response.EnsureSuccessStatusCode();
 
         var contentStream = await response.Content.ReadAsStreamAsync();
-        var createChatResponse = await JsonSerializer.DeserializeAsync<CreateChatResponse>(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+        var createChatResponse = await JsonSerializer.DeserializeAsync<CreateChatResponse>(contentStream, jsOpts);
         Assert.NotNull(createChatResponse);
 
         // Ask something to the bot
@@ -37,12 +37,12 @@ public async void ChatMessagePostSucceedsWithValidInput()
         response.EnsureSuccessStatusCode();
 
         contentStream = await response.Content.ReadAsStreamAsync();
-        var askResult = await JsonSerializer.DeserializeAsync<AskResult>(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+        var askResult = await JsonSerializer.DeserializeAsync<AskResult>(contentStream, jsOpts);
         Assert.NotNull(askResult);
         Assert.False(string.IsNullOrEmpty(askResult.Value));
 
         // Clean up
-        response = await this.HTTPClient.DeleteAsync($"chats/{createChatResponse.ChatSession.Id}");
+        response = await this.HTTPClient.DeleteAsync($"chats/{createChatResponse.ChatSession.Id}").ConfigureAwait(false);
         response.EnsureSuccessStatusCode();
     }
 }
diff --git a/integration-tests/HealthzTests.cs b/integration-tests/HealthzTests.cs
@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System.Net.Http;
 using Xunit;
 
 namespace ChatCopilotIntegrationTests;

diff --git a/integration-tests/ServiceInfoTests.cs b/integration-tests/ServiceInfoTests.cs
@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System.Net.Http;
 using System.Text.Json;
 using CopilotChat.WebApi.Models.Response;
 using CopilotChat.WebApi.Options;
@@ -10,6 +9,8 @@ namespace ChatCopilotIntegrationTests;
 
 public class ServiceInfoTests : ChatCopilotIntegrationTest
 {
+    private static readonly JsonSerializerOptions jsonOptions = new() { PropertyNameCaseInsensitive = true };
+
     [Fact]
     public async void GetServiceInfo()
     {
@@ -19,7 +20,7 @@ public async void GetServiceInfo()
         response.EnsureSuccessStatusCode();
 
         var contentStream = await response.Content.ReadAsStreamAsync();
-        var objectFromResponse = await JsonSerializer.DeserializeAsync<ServiceInfoResponse>(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+        var objectFromResponse = await JsonSerializer.DeserializeAsync<ServiceInfoResponse>(contentStream, jsonOptions);
 
         Assert.NotNull(objectFromResponse);
         Assert.False(string.IsNullOrEmpty(objectFromResponse.MemoryStore.SelectedType));
@@ -33,7 +34,7 @@ public async void GetAuthConfig()
         response.EnsureSuccessStatusCode();
 
         var contentStream = await response.Content.ReadAsStreamAsync();
-        var objectFromResponse = await JsonSerializer.DeserializeAsync<FrontendAuthConfig>(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+        var objectFromResponse = await JsonSerializer.DeserializeAsync<FrontendAuthConfig>(contentStream, jsonOptions);
 
         Assert.NotNull(objectFromResponse);
         Assert.Equal(ChatAuthenticationOptions.AuthenticationType.AzureAd.ToString(), objectFromResponse.AuthType);

diff --git a/integration-tests/SpeechTokenTests.cs b/integration-tests/SpeechTokenTests.cs
@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System.Net.Http;
 using System.Text.Json;
 using CopilotChat.WebApi.Models.Response;
 using Xunit;
@@ -9,6 +8,8 @@ namespace ChatCopilotIntegrationTests;
 
 public class SpeechTokenTests : ChatCopilotIntegrationTest
 {
+    private static readonly JsonSerializerOptions jsonOpts = new() { PropertyNameCaseInsensitive = true };
+
     [Fact]
     public async void GetSpeechToken()
     {
@@ -18,7 +19,7 @@ public async void GetSpeechToken()
         response.EnsureSuccessStatusCode();
 
         var contentStream = await response.Content.ReadAsStreamAsync();
-        var speechTokenResponse = await JsonSerializer.DeserializeAsync<SpeechTokenResponse>(contentStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+        var speechTokenResponse = await JsonSerializer.DeserializeAsync<SpeechTokenResponse>(contentStream, jsonOpts);
 
         Assert.NotNull(speechTokenResponse);
         Assert.True((speechTokenResponse.IsSuccess == true && !string.IsNullOrEmpty(speechTokenResponse.Token)) ||

diff --git a/integration-tests/StaticFiles.cs b/integration-tests/StaticFiles.cs
@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System.Net.Http;
 using Xunit;
 
 namespace ChatCopilotIntegrationTests;