Skip to content

Commit

Permalink
Add enterprise file search (#47614)
Browse files Browse the repository at this point in the history
* Add enterprise and batch file search.

* Fix
  • Loading branch information
nick863 authored Dec 19, 2024
1 parent 9c6f9c6 commit e3602f9
Show file tree
Hide file tree
Showing 13 changed files with 772 additions and 3 deletions.
104 changes: 104 additions & 0 deletions sdk/ai/Azure.AI.Projects/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ Use the AI Projects client library to:
- [Create and execute run](#create-and-execute-run)
- [Retrieve messages](#retrieve-messages)
- [File search](#file-search)
- [Enterprise File Search](#create-agent-with-enterprise-file-search)
- [Code interpreter attachment](#create-message-with-code-interpreter-attachment)
- [Function call](#function-call)
- [Azure function call](#azure-function-call)
- [Azure Function Call](#create-agent-with-azure-function-call)
Expand Down Expand Up @@ -210,6 +212,108 @@ Agent agent = agentResponse.Value;
With a file ID association and a supported tool enabled, the agent will then be able to consume the associated
data when running threads.

#### Create Agent with Enterprise File Search

We can upload file to Azure as it is shown in the example, or use the existing Azure blob storage. In the code below we demonstrate how this can be achieved. First we upload file to azure and create `VectorStoreDataSource`, which then is used to create vector store. This vector store is then given to the `FileSearchTool` constructor.

```C# Snippet:CreateVectorStoreBlob
var ds = new VectorStoreDataSource(
assetIdentifier: blobURI,
assetType: VectorStoreDataSourceAssetType.UriAsset
);
var vectorStoreTask = await client.CreateVectorStoreAsync(
name: "sample_vector_store",
storeConfiguration: new VectorStoreConfiguration(
dataSources: new List<VectorStoreDataSource> { ds }
)
);
var vectorStore = vectorStoreTask.Value;

FileSearchToolResource fileSearchResource = new([vectorStore.Id], null);

List<ToolDefinition> tools = [new FileSearchToolDefinition()];
Response<Agent> agentResponse = await client.CreateAgentAsync(
model: modelName,
name: "my-assistant",
instructions: "You are helpful assistant.",
tools: tools,
toolResources: new ToolResources() { FileSearch = fileSearchResource }
);
```

We also can attach files to the existing vector store. In the code snippet below, we first create an empty vector store and add file to it.

```C# Snippet:BatchFileAttachment
var ds = new VectorStoreDataSource(
assetIdentifier: blobURI,
assetType: VectorStoreDataSourceAssetType.UriAsset
);
var vectorStoreTask = await client.CreateVectorStoreAsync(
name: "sample_vector_store"
);
var vectorStore = vectorStoreTask.Value;

var uploadTask = await client.CreateVectorStoreFileBatchAsync(
vectorStoreId: vectorStore.Id,
dataSources: new List<VectorStoreDataSource> { ds }
);
Console.WriteLine($"Created vector store file batch, vector store file batch ID: {uploadTask.Value.Id}");

FileSearchToolResource fileSearchResource = new([vectorStore.Id], null);
```

#### Create Message with Code Interpreter Attachment

To attach a file with the context to the message, use the `MessageAttachment` class. To be able to process the attached file contents we need to provide the `List` with the single element `CodeInterpreterToolDefinition` as a `tools` parameter to both `CreateAgent` method and `MessageAttachment` class constructor.

Here is an example to pass `CodeInterpreterTool` as tool:

```C# Snippet:CreateAgentWithInterpreterTool
AgentsClient client = new AgentsClient(connectionString, new DefaultAzureCredential());

List<ToolDefinition> tools = [ new CodeInterpreterToolDefinition() ];
Response<Agent> agentResponse = await client.CreateAgentAsync(
model: modelName,
name: "my-assistant",
instructions: "You are helpful assistant.",
tools: tools
);
Agent agent = agentResponse.Value;

var fileResponse = await client.UploadFileAsync(filePath, AgentFilePurpose.Agents);
var fileId = fileResponse.Value.Id;

var attachment = new MessageAttachment(
fileId: fileId,
tools: tools
);

Response<AgentThread> threadResponse = await client.CreateThreadAsync();
AgentThread thread = threadResponse.Value;

Response<ThreadMessage> messageResponse = await client.CreateMessageAsync(
threadId: thread.Id,
role: MessageRole.User,
content: "What does the attachment say?",
attachments: new List< MessageAttachment > { attachment}
);
ThreadMessage message = messageResponse.Value;
```

Azure blob storage can be used as a message attachment. In this case, use `VectorStoreDataSource` as a data source:

```C# Snippet:CreateMessageAttachmentWithBlobStore
var ds = new VectorStoreDataSource(
assetIdentifier: blobURI,
assetType: VectorStoreDataSourceAssetType.UriAsset
);

var attachment = new MessageAttachment(
ds: ds,
tools: tools
);
```

#### Function call

Tools that reference caller-defined capabilities as functions can be provided to an agent to allow it to
Expand Down
3 changes: 3 additions & 0 deletions sdk/ai/Azure.AI.Projects/api/Azure.AI.Projects.net8.0.cs
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,7 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
public partial class FileSearchToolResource : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.FileSearchToolResource>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.FileSearchToolResource>
{
public FileSearchToolResource() { }
public FileSearchToolResource(System.Collections.Generic.IList<string> vectorStoreIds, System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> vectorStores) { }
public System.Collections.Generic.IList<string> VectorStoreIds { get { throw null; } }
public System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> VectorStores { get { throw null; } }
protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { }
Expand Down Expand Up @@ -1151,7 +1152,9 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
}
public partial class MessageAttachment : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.MessageAttachment>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.MessageAttachment>
{
public MessageAttachment(Azure.AI.Projects.VectorStoreDataSource ds, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
public MessageAttachment(System.Collections.Generic.IEnumerable<System.BinaryData> tools) { }
public MessageAttachment(string fileId, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
public Azure.AI.Projects.VectorStoreDataSource DataSource { get { throw null; } set { } }
public string FileId { get { throw null; } set { } }
public System.Collections.Generic.IList<System.BinaryData> Tools { get { throw null; } }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,7 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
public partial class FileSearchToolResource : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.FileSearchToolResource>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.FileSearchToolResource>
{
public FileSearchToolResource() { }
public FileSearchToolResource(System.Collections.Generic.IList<string> vectorStoreIds, System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> vectorStores) { }
public System.Collections.Generic.IList<string> VectorStoreIds { get { throw null; } }
public System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> VectorStores { get { throw null; } }
protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { }
Expand Down Expand Up @@ -1151,7 +1152,9 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
}
public partial class MessageAttachment : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.MessageAttachment>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.MessageAttachment>
{
public MessageAttachment(Azure.AI.Projects.VectorStoreDataSource ds, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
public MessageAttachment(System.Collections.Generic.IEnumerable<System.BinaryData> tools) { }
public MessageAttachment(string fileId, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
public Azure.AI.Projects.VectorStoreDataSource DataSource { get { throw null; } set { } }
public string FileId { get { throw null; } set { } }
public System.Collections.Generic.IList<System.BinaryData> Tools { get { throw null; } }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System.Collections.Generic;

namespace Azure.AI.Projects
{
public partial class FileSearchToolResource
{
public FileSearchToolResource(
IList<string> vectorStoreIds,
IList<VectorStoreConfigurations> vectorStores
)
{
VectorStoreIds = vectorStoreIds;
if (vectorStores == null)
VectorStores = new ChangeTrackingList<VectorStoreConfigurations>();
else
VectorStores = vectorStores;
}
}
}
44 changes: 44 additions & 0 deletions sdk/ai/Azure.AI.Projects/src/Custom/Agent/MessageAttachment.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.ClientModel.Primitives;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.Json;
namespace Azure.AI.Projects;

public partial class MessageAttachment
{
public MessageAttachment(VectorStoreDataSource ds, List<ToolDefinition> tools)
{
FileId = null;
DataSource = ds;
Tools = serializeJson(tools);
_serializedAdditionalRawData = null;
}

public MessageAttachment(string fileId, List<ToolDefinition> tools)
{
FileId = fileId;
DataSource = null;
Tools = serializeJson(tools);
_serializedAdditionalRawData = null;
}

private static List<BinaryData> serializeJson<T>(List<T> definitions) where T: IJsonModel<T>
{
List<BinaryData> serializedDefinitions = new();
foreach (IJsonModel<T> definition in definitions)
{
var stream = new MemoryStream();
var writer = new Utf8JsonWriter(stream);
definition.Write(writer, ModelReaderWriterOptions.Json);
writer.Flush();
string json = Encoding.UTF8.GetString(stream.ToArray());
serializedDefinitions.Add(new BinaryData(json));
}
return serializedDefinitions;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@ public class AIProjectsTestEnvironment : TestEnvironment
public string BINGCONNECTIONNAME => GetRecordedVariable("BING_CONNECTION_NAME");
public string MODELDEPLOYMENTNAME => GetRecordedVariable("MODEL_DEPLOYMENT_NAME");
public string STORAGE_QUEUE_URI => GetRecordedVariable("STORAGE_QUEUE_URI");
public string AZURE_BLOB_URI => GetRecordedVariable("AZURE_BLOB_URI");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.Net.Mail;
using System.Threading;
using System.Threading.Tasks;
using Azure.Core.TestFramework;
using NUnit.Framework;
using NUnit.Framework.Internal.Execution;

namespace Azure.AI.Projects.Tests;

public partial class Sample_Agent_Enterprise_File_Search : SamplesBase<AIProjectsTestEnvironment>
{
[Test]
public async Task EnterpriseFileSearch()
{
var connectionString = TestEnvironment.AzureAICONNECTIONSTRING;
// For now we will take the File URI from the environment variables.
// In future we may want to upload file to Azure here.
var blobURI = TestEnvironment.AZURE_BLOB_URI;
var modelName = TestEnvironment.MODELDEPLOYMENTNAME;
AgentsClient client = new AgentsClient(connectionString, new DefaultAzureCredential());

#region Snippet:CreateVectorStoreBlob
var ds = new VectorStoreDataSource(
assetIdentifier: blobURI,
assetType: VectorStoreDataSourceAssetType.UriAsset
);
var vectorStoreTask = await client.CreateVectorStoreAsync(
name: "sample_vector_store",
storeConfiguration: new VectorStoreConfiguration(
dataSources: new List<VectorStoreDataSource> { ds }
)
);
var vectorStore = vectorStoreTask.Value;

FileSearchToolResource fileSearchResource = new([vectorStore.Id], null);

List<ToolDefinition> tools = [new FileSearchToolDefinition()];
Response<Agent> agentResponse = await client.CreateAgentAsync(
model: modelName,
name: "my-assistant",
instructions: "You are helpful assistant.",
tools: tools,
toolResources: new ToolResources() { FileSearch = fileSearchResource }
);
#endregion
Response<AgentThread> threadResponse = await client.CreateThreadAsync();
AgentThread thread = threadResponse.Value;

Response<ThreadMessage> messageResponse = await client.CreateMessageAsync(
threadId: thread.Id,
role: MessageRole.User,
content: "What feature does Smart Eyewear offer?"
);
ThreadMessage message = messageResponse.Value;

Response<ThreadRun> runResponse = await client.CreateRunAsync(
thread.Id,
agentResponse.Value.Id
);
ThreadRun run = runResponse.Value;

do
{
await Task.Delay(TimeSpan.FromMilliseconds(500));
runResponse = await client.GetRunAsync(thread.Id, runResponse.Value.Id);
}
while (runResponse.Value.Status == RunStatus.Queued
|| runResponse.Value.Status == RunStatus.InProgress);

Response<PageableList<ThreadMessage>> afterRunMessagesResponse
= await client.GetMessagesAsync(thread.Id);
IReadOnlyList<ThreadMessage> messages = afterRunMessagesResponse.Value.Data;
WriteMessages(messages);

var delTask = await client.DeleteVectorStoreAsync(vectorStore.Id);
if (delTask.Value.Deleted)
{
Console.WriteLine($"Deleted vector store {vectorStore.Id}");
}
else
{
Console.WriteLine($"Unable to delete vector store {vectorStore.Id}");
}
await client.DeleteAgentAsync(agentResponse.Value.Id);
}

private void WriteMessages(IEnumerable<ThreadMessage> messages)
{
foreach (ThreadMessage threadMessage in messages)
{
Console.Write($"{threadMessage.CreatedAt:yyyy-MM-dd HH:mm:ss} - {threadMessage.Role,10}: ");
foreach (MessageContent contentItem in threadMessage.ContentItems)
{
if (contentItem is MessageTextContent textItem)
{
Console.Write(textItem.Text);
}
else if (contentItem is MessageImageFileContent imageFileItem)
{
Console.Write($"<image from ID: {imageFileItem.FileId}");
}
Console.WriteLine();
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,9 @@
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text.Json.Serialization;
using System.Text.Json;
using System.Threading.Tasks;
using Azure.Core.TestFramework;
using NUnit.Framework;
using Newtonsoft.Json.Linq;

namespace Azure.AI.Projects.Tests;

Expand Down
Loading

0 comments on commit e3602f9

Please sign in to comment.