Checkpoint

microsoft · Aug 11, 2023 · ab59840 · ab59840
1 parent 889d698
commit ab59840
Show file tree

Hide file tree

Showing 9 changed files with 107 additions and 78 deletions.
diff --git a/memorypipeline/BuilderExtensions.cs b/memorypipeline/BuilderExtensions.cs
@@ -17,7 +17,6 @@
 using Microsoft.SemanticMemory.Core.Pipeline.Queue;
 using Microsoft.SemanticMemory.Core.Pipeline.Queue.AzureQueues;
 using Microsoft.SemanticMemory.Core.Pipeline.Queue.FileBasedQueues;
-using Microsoft.SemanticMemory.Core.Pipeline.Queue.RabbitMq;
 using Microsoft.SemanticMemory.Core.Pipeline;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
 
@@ -98,12 +97,6 @@ private static void ConfigureQueueSystem(this WebApplicationBuilder builder, Sem
                     .Get<AzureQueueConfig>()!);
                 break;
 
-            case string y when y.Equals("RabbitMQ", StringComparison.OrdinalIgnoreCase):
-                builder.Services.AddRabbitMq(builder.Configuration
-                    .GetSection(ConfigRoot).GetSection("Services").GetSection("RabbitMq")
-                    .Get<RabbitMqConfig>()!);
-                break;
-
             case string y when y.Equals("FileBasedQueue", StringComparison.OrdinalIgnoreCase):
                 builder.Services.AddFileBasedQueue(builder.Configuration
                     .GetSection(ConfigRoot).GetSection("Services").GetSection("FileBasedQueue")

diff --git a/memorypipeline/CopilotChatMemoryPipeline.csproj b/memorypipeline/CopilotChatMemoryPipeline.csproj
@@ -9,7 +9,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.SemanticMemory.Core" Version="0.0.5.1-preview" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="0.19.230804.2-preview" />
+    <PackageReference Include="Microsoft.SemanticMemory.Core" Version="0.0.9.1-preview" />
   </ItemGroup>
 
 </Project>
diff --git a/memorypipeline/appsettings.json b/memorypipeline/appsettings.json
@@ -4,9 +4,10 @@
     "ContentStorageType": "FileSystemContentStorage",
     // Data ingestion pipelines configuration.
     "DataIngestion": {
+      "OrchestrationType": "Distributed",
       "DistributedOrchestration": {
-        // "AzureQueue", "RabbitMQ", "FileBasedQueue"
-        "QueueType": "FileBasedQueue"
+        // "AzureQueue", "FileBasedQueue"
+        "QueueType": "AzureQueue"
       },
       // Multiple generators can be used, e.g. for data migration, A/B testing, etc.
       "EmbeddingGeneratorTypes": [
@@ -19,10 +20,10 @@
     },
     "Services": {
       "AzureBlobs": {
-        // "ConnectionString" or "AzureIdentity"
+        // "ConnectionString" or "AzureIdentity".
         // AzureIdentity: use automatic AAD authentication mechanism. You can test locally
         //   using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET.
-        "Auth": "AzureIdentity",
+        "Auth": "ConnectionString",
         // Azure Storage account name, required when using AzureIdentity auth
         // Note: you can use an env var 'SemanticMemory__Services__AzureBlobs__Account' to set this
         "Account": "",
@@ -35,9 +36,10 @@
         "EndpointSuffix": "core.windows.net"
       },
       "AzureQueue": {
-        // - AzureIdentity: use automatic AAD authentication mechanism
-        // - ConnectionString: auth using a connection string
-        "Auth": "AzureIdentity",
+        // "ConnectionString" or "AzureIdentity". For other options see <AzureQueueConfig>.
+        // AzureIdentity: use automatic AAD authentication mechanism. You can test locally
+        //   using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET.
+        "Auth": "ConnectionString",
         // Azure Storage account name, required when using AzureIdentity auth
         // Note: you can use an env var 'SemanticMemory__Orchestration__DistributedPipeline__AzureQueue__Account' to set this
         "Account": "",
@@ -48,22 +50,22 @@
         "EndpointSuffix": "core.windows.net"
       },
       "AzureCognitiveSearch": {
-        // "ApiKey" or "AzureIdentity"
+        // "ApiKey" or "AzureIdentity". For other options see <AzureCognitiveSearchConfig>.
         // AzureIdentity: use automatic AAD authentication mechanism. You can test locally
         //   using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET.
         "Auth": "ApiKey",
         "Endpoint": "https://<...>",
         "APIKey": "",
-        "VectorIndexPrefix": "smemory-",
+        "VectorIndexPrefix": "smemory-"
       },
       "AzureOpenAIEmbedding": {
         // "ApiKey" or "AzureIdentity"
         // AzureIdentity: use automatic AAD authentication mechanism. You can test locally
         //   using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET.
         "Auth": "ApiKey",
         "Endpoint": "https://<...>.openai.azure.com/",
-        "Deployment": "",
-        "APIKey": "",
+        "Deployment": "text-embedding-ada-002",
+        "APIKey": ""
       },
       "FileSystemContentStorage": {
         "Directory": "/tmp/semanticmemory/content"
@@ -76,19 +78,13 @@
       "OpenAI": {
         "EmbeddingModel": "text-embedding-ada-002",
         "APIKey": "",
-        "OrgId": "",
-      },
-      "RabbitMq": {
-        "Host": "127.0.0.1",
-        "Port": "5672",
-        "Username": "user",
-        "Password": ""
+        "OrgId": ""
       },
       "FileBasedQueue": {
         "Path": "/tmp/semanticmemory/queues",
         "CreateIfNotExist": true
-      },
-    },
+      }
+    }
   },
   "Logging": {
     "LogLevel": {

diff --git a/webapi/Controllers/DocumentController.cs b/webapi/Controllers/DocumentController.cs
@@ -17,6 +17,7 @@
 using Microsoft.AspNetCore.SignalR;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Microsoft.SemanticMemory.Core;
 using Microsoft.SemanticMemory.Core.Pipeline;
 using Microsoft.SemanticMemory.Core.WebService;
 
@@ -68,7 +69,7 @@ public DocumentController(
     [ProducesResponseType(StatusCodes.Status200OK)]
     [ProducesResponseType(StatusCodes.Status400BadRequest)]
     public async Task<IActionResult> DocumentStatusAsync(
-        [FromServices] IPipelineOrchestrator orchestrator,
+        [FromServices] ISemanticMemoryService memoryService,
         [FromServices] IHubContext<MessageRelayHub> messageRelayHubContext,
         [FromBody] DocumentStatusForm documentStatusForm)
     {
@@ -110,7 +111,7 @@ async IAsyncEnumerable<StatusResult> QueryAsync()
         {
             foreach (var documentReference in documentStatusForm.FileReferences)
             {
-                DataPipeline? pipeline = await orchestrator.ReadPipelineStatusAsync(targetCollectionName, documentReference);
+                DataPipeline? pipeline = await memoryService.ReadPipelineStatusAsync(targetCollectionName, documentReference);
                 if (pipeline == null)
                 {
                     yield return new StatusResult(targetCollectionName, documentReference);
@@ -140,7 +141,7 @@ async IAsyncEnumerable<StatusResult> QueryAsync()
     [ProducesResponseType(StatusCodes.Status200OK)]
     [ProducesResponseType(StatusCodes.Status400BadRequest)]
     public async Task<IActionResult> DocumentImportAsync(
-        [FromServices] IPipelineOrchestrator orchestrator,
+        [FromServices] ISemanticMemoryService memoryService,
         [FromServices] IHubContext<MessageRelayHub> messageRelayHubContext,
         [FromForm] DocumentImportForm documentImportForm)
     {
@@ -196,7 +197,7 @@ async IAsyncEnumerable<ImportResult> ImportAsync()
         {
             foreach (var formFile in documentImportForm.FormFiles)
             {
-                var importResult = await this.ImportDocumentHelperAsync(orchestrator, formFile, documentImportForm);
+                var importResult = await this.ImportDocumentHelperAsync(memoryService, formFile, documentImportForm);
                 documentMessageContent.AddDocument(
                     formFile.FileName,
                     this.GetReadableByteString(formFile.Length),
@@ -408,7 +409,7 @@ private async Task ValidateDocumentStatusFormAsync(DocumentStatusForm documentSt
     /// <param name="formFile">The form file.</param>
     /// <param name="documentImportForm">The document import form.</param>
     /// <returns>Import result.</returns>
-    private async Task<ImportResult> ImportDocumentHelperAsync(IPipelineOrchestrator orchestrator, IFormFile formFile, DocumentImportForm documentImportForm)
+    private async Task<ImportResult> ImportDocumentHelperAsync(ISemanticMemoryService memoryService, IFormFile formFile, DocumentImportForm documentImportForm)
     {
         this._logger.LogInformation("Importing document {0}", formFile.FileName);
 
@@ -427,7 +428,7 @@ private async Task<ImportResult> ImportDocumentHelperAsync(IPipelineOrchestrator
             // Tags = null $$$ ???
         };
 
-        await orchestrator.UploadFileAsync(uploadRequest);
+        await memoryService.UploadFileAsync(uploadRequest);
 
         var importResult = new ImportResult(memorySource.Id);
 

diff --git a/webapi/CopilotChatWebApi.csproj b/webapi/CopilotChatWebApi.csproj
@@ -21,7 +21,7 @@
     <PackageReference Include="Microsoft.SemanticKernel.Skills.MsGraph" Version="0.19.230804.2-preview" />
     <PackageReference Include="Microsoft.SemanticKernel.Skills.OpenAPI" Version="0.19.230804.2-preview" />
     <PackageReference Include="Microsoft.SemanticKernel.Skills.Web" Version="0.19.230804.2-preview" />
-    <PackageReference Include="Microsoft.SemanticMemory.Core" Version="0.0.5.1-preview" />
+    <PackageReference Include="Microsoft.SemanticMemory.Core" Version="0.0.9.1-preview" />
     <PackageReference Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.2.2" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
     <PackageReference Include="Microsoft.Identity.Web" Version="2.13.2" />

diff --git a/webapi/Extensions/SemanticMemoryExtensions.cs b/webapi/Extensions/SemanticMemoryExtensions.cs
@@ -4,6 +4,7 @@
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using Microsoft.SemanticMemory.Core.AppBuilders;
 using Microsoft.SemanticMemory.Core.Configuration;
 using Microsoft.SemanticMemory.Core.ContentStorage.AzureBlobs;
@@ -13,8 +14,10 @@
 using Microsoft.SemanticMemory.Core.Pipeline.Queue;
 using Microsoft.SemanticMemory.Core.Pipeline.Queue.AzureQueues;
 using Microsoft.SemanticMemory.Core.Pipeline.Queue.FileBasedQueues;
-using Microsoft.SemanticMemory.Core.Pipeline.Queue.RabbitMq;
-using Microsoft.Extensions.Options;
+using Microsoft.SemanticMemory.Core.AI.AzureOpenAI;
+using Microsoft.SemanticMemory.Core.AI.OpenAI;
+using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
+using Microsoft.SemanticKernel.AI.Embeddings;
 
 namespace CopilotChat.WebApi.Extensions;
 
@@ -28,6 +31,9 @@ internal static class SemanticMemoryExtensions
     /// <summary>
     /// Add Semantic Memory services
     /// </summary>
+    /// <remarks>
+    /// Forced to conform with the current state of semantic-memory.
+    /// </remarks>
     public static void AddSemanticMemoryServices(this WebApplicationBuilder builder)
     {
         var memoryConfig = builder.AddSemanticMemoryOptions();
@@ -37,7 +43,8 @@ public static void AddSemanticMemoryServices(this WebApplicationBuilder builder)
         builder
             .ConfigureContentStorage(memoryConfig)
             .ConfigureQueueSystem(memoryConfig)
-            .ConfigureEmbeddingStorage(memoryConfig);
+            .ConfigureEmbeddingStorage(memoryConfig)
+            .ConfigureAI(memoryConfig);
     }
 
     /// <summary>
@@ -97,12 +104,6 @@ private static WebApplicationBuilder ConfigureQueueSystem(this WebApplicationBui
                     .Get<AzureQueueConfig>()!);
                 break;
 
-            case string y when y.Equals("RabbitMQ", StringComparison.OrdinalIgnoreCase):
-                builder.Services.AddRabbitMq(builder.Configuration
-                    .GetSection(ConfigRoot).GetSection("Services").GetSection("RabbitMq")
-                    .Get<RabbitMqConfig>()!);
-                break;
-
             case string y when y.Equals("FileBasedQueue", StringComparison.OrdinalIgnoreCase):
                 builder.Services.AddFileBasedQueue(builder.Configuration
                     .GetSection(ConfigRoot).GetSection("Services").GetSection("FileBasedQueue")
@@ -141,4 +142,52 @@ private static WebApplicationBuilder ConfigureEmbeddingStorage(this WebApplicati
 
         return builder;
     }
+
+    private static WebApplicationBuilder ConfigureAI(this WebApplicationBuilder builder, SemanticMemoryConfig config)
+    {
+        var embeddingGenerationServices = new TypeCollection<ITextEmbeddingGeneration>();
+        builder.Services.AddSingleton(embeddingGenerationServices);
+
+        switch (config.Retrieval.EmbeddingGeneratorType)
+        {
+            case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase):
+            case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase):
+                embeddingGenerationServices.Add<AzureTextEmbeddingGeneration>();
+                builder.Services.AddAzureOpenAIEmbeddingGeneration(builder.Configuration
+                    .GetSection(ConfigRoot).GetSection("Services").GetSection("AzureOpenAIEmbedding")
+                    .Get<AzureOpenAIConfig>()!);
+                break;
+
+            case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase):
+                embeddingGenerationServices.Add<OpenAITextEmbeddingGeneration>();
+                builder.Services.AddOpenAITextEmbeddingGeneration(builder.Configuration
+                    .GetSection(ConfigRoot).GetSection("Services").GetSection("OpenAI")
+                    .Get<OpenAIConfig>()!);
+                break;
+
+            default:
+                throw new NotSupportedException($"Unknown/unsupported {config.Retrieval.EmbeddingGeneratorType} text generator");
+        }
+
+        switch (config.Retrieval.TextGeneratorType)
+        {
+            case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase):
+            case string y when y.Equals("AzureOpenAIText", StringComparison.OrdinalIgnoreCase):
+                builder.Services.AddAzureOpenAITextGeneration(builder.Configuration
+                    .GetSection(ConfigRoot).GetSection("Services").GetSection("AzureOpenAIText")
+                    .Get<AzureOpenAIConfig>()!);
+                break;
+
+            case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase):
+                builder.Services.AddOpenAITextGeneration(builder.Configuration
+                    .GetSection(ConfigRoot).GetSection("Services").GetSection("OpenAI")
+                    .Get<OpenAIConfig>()!);
+                break;
+
+            default:
+                throw new NotSupportedException($"Unknown/unsupported {config.Retrieval.TextGeneratorType} text generator");
+        }
+
+        return builder;
+    }
 }
diff --git a/webapi/Extensions/ServiceExtensions.cs b/webapi/Extensions/ServiceExtensions.cs
@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Reflection;
 using Azure;
 using CopilotChat.WebApi.Auth;
@@ -16,7 +17,6 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using Microsoft.Identity.Web;
-using Microsoft.SemanticMemory.Core.Configuration;
 using Tesseract;
 
 namespace CopilotChat.WebApi.Extensions;
@@ -272,6 +272,12 @@ private static void TrimStringProperties<T>(T options) where T : class
                     continue;
                 }
 
+                // Skip index properties
+                if (property.GetIndexParameters().Length == 0)
+                {
+                    continue;
+                }
+
                 // Property is a built-in type, readable, and writable.
                 if (property.PropertyType.Namespace == "System" &&
                     property.CanRead &&

diff --git a/webapi/Program.cs b/webapi/Program.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using CopilotChat.WebApi.Extensions;
@@ -32,8 +33,6 @@ public static async Task Main(string[] args)
     {
         WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
 
-
-
         // Load in configuration settings from appsettings.json, user-secrets, key vaults, etc...
         builder.Host.AddConfiguration();
         builder.WebHost.UseUrls(); // Disables endpoint override warning message when using IConfiguration for Kestrel endpoint.
@@ -61,9 +60,7 @@ public static async Task Main(string[] args)
             .AddLogging(logBuilder => logBuilder.AddApplicationInsights())
             .AddSingleton<ITelemetryService, AppInsightsTelemetryService>();
 
-#if DEBUG
-        TelemetryDebugWriter.IsTracingDisabled = false;
-#endif
+        TelemetryDebugWriter.IsTracingDisabled = Debugger.IsAttached;
 
         // Add in the rest of the services.
         builder.Services