From e4720bc6cf0e225616529412991393da5827afde Mon Sep 17 00:00:00 2001 From: Mark Sze <66362098+marklysze@users.noreply.github.com> Date: Sat, 22 Jun 2024 12:24:23 +1000 Subject: [PATCH 01/57] Message handling to support multiple function calls (#2997) --- autogen/oai/anthropic.py | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/autogen/oai/anthropic.py b/autogen/oai/anthropic.py index 9faa4e2cb808..e2448929e618 100644 --- a/autogen/oai/anthropic.py +++ b/autogen/oai/anthropic.py @@ -49,10 +49,10 @@ "claude-3-5-sonnet-20240620": (0.003, 0.015), "claude-3-sonnet-20240229": (0.003, 0.015), "claude-3-opus-20240229": (0.015, 0.075), - "claude-2.0": (0.008, 0.024), + "claude-3-haiku-20240307": (0.00025, 0.00125), "claude-2.1": (0.008, 0.024), - "claude-3.0-opus": (0.015, 0.075), - "claude-3.0-haiku": (0.00025, 0.00125), + "claude-2.0": (0.008, 0.024), + "claude-instant-1.2": (0.008, 0.024), } @@ -250,6 +250,7 @@ def oai_messages_to_anthropic_messages(params: Dict[str, Any]) -> list[dict[str, tool_use_messages = 0 tool_result_messages = 0 last_tool_use_index = -1 + last_tool_result_index = -1 for message in params["messages"]: if message["role"] == "system": params["system"] = message["content"] @@ -290,25 +291,26 @@ def oai_messages_to_anthropic_messages(params: Dict[str, Any]) -> list[dict[str, } ) elif "tool_call_id" in message: - - if expected_role == "assistant": - # Insert an extra assistant message as we will append a user message - processed_messages.append(assistant_continue_message) - if has_tools: # Map the tool usage call to tool_result for Anthropic - processed_messages.append( - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": message["tool_call_id"], - "content": message["content"], - } - ], - } - ) + tool_result = { + "type": "tool_result", + "tool_use_id": message["tool_call_id"], + "content": message["content"], + } + + # If the previous message also had a tool_result, add it to that + # Otherwise append a new message + if last_tool_result_index == len(processed_messages) - 1: + processed_messages[-1]["content"].append(tool_result) + else: + if expected_role == "assistant": + # Insert an extra assistant message as we will append a user message + processed_messages.append(assistant_continue_message) + + processed_messages.append({"role": "user", "content": [tool_result]}) + last_tool_result_index = len(processed_messages) - 1 + tool_result_messages += 1 else: # Not using tools, so put in a plain text message From 9b92093b00b9c4e15c3dfc91f68b2a6ffb80191e Mon Sep 17 00:00:00 2001 From: Alex Reibman Date: Fri, 21 Jun 2024 20:05:11 -0700 Subject: [PATCH 02/57] LLM Observability documentation fixes: Broken links, grammar, and spelling (#2995) * update markdown hyperlinks to stable urls * update notebook images and text * re-write observability section * Updated section * update wording * added newline * update styling in image tags to be jsx compatible * added text * update link * simplified text --------- Co-authored-by: Braelyn Boynton --- notebook/agentchat_agentops.ipynb | 185 ++++++++++++----------- website/docs/ecosystem/agentops.md | 28 ++-- website/docs/topics/llm-observability.md | 55 +++---- 3 files changed, 138 insertions(+), 130 deletions(-) diff --git a/notebook/agentchat_agentops.ipynb b/notebook/agentchat_agentops.ipynb index 293efa8e4bd6..71106e45d3c4 100644 --- a/notebook/agentchat_agentops.ipynb +++ b/notebook/agentchat_agentops.ipynb @@ -7,7 +7,7 @@ "collapsed": false }, "source": [ - "# AgentOps" + "# Agent Tracking with AgentOps" ] }, { @@ -15,7 +15,7 @@ "id": "a447802c88c8a240", "metadata": {}, "source": [ - "![logo](https://raw.githubusercontent.com/AgentOps-AI/agentops/35d5682866921a9e28d8ef66ae3c3b3d92d8fa6b/img/logo.png)\n", + "\n", "\n", "[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for AI agents.\n", "\n", @@ -27,8 +27,11 @@ "id": "b354c068", "metadata": {}, "source": [ - "### Dashboard\n", - "![Agent Dashboard](https://github.com/AgentOps-AI/agentops/assets/14807319/158e082a-9a7d-49b7-9b41-51a49a1f7d3d)" + "### Overview Dashboard\n", + "\n", + "\n", + "### Session Replays\n", + "" ] }, { @@ -39,7 +42,7 @@ "## Adding AgentOps to an existing Autogen service.\n", "To get started, you'll need to install the AgentOps package and set an API key.\n", "\n", - "AgentOps automatically configures itself when it's initialized. This means your agents will be tracked and logged to your AgentOps account right away." + "AgentOps automatically configures itself when it's initialized meaning your agent run data will be tracked and logged to your AgentOps account right away." ] }, { @@ -69,7 +72,7 @@ "\n", "By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Alternatively, you can pass one in as an optional parameter.\n", "\n", - "Create an account and API key at [AgentOps.ai](https://agentops.ai/)" + "Create an account and obtain an API key at [AgentOps.ai](https://agentops.ai/settings/projects)" ] }, { @@ -87,12 +90,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001B[0m\u001B[0m\n" + "๐Ÿ–‡ AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001b[0m\u001b[0m\n" ] }, { "data": { - "text/plain": "UUID('8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4')" + "text/plain": [ + "UUID('8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4')" + ] }, "execution_count": 1, "metadata": {}, @@ -104,7 +109,7 @@ "\n", "from autogen import ConversableAgent, UserProxyAgent, config_list_from_json\n", "\n", - "agentops.init(api_key=\"7c94212b-b89d-47a6-a20c-23b2077d3226\") # or agentops.init(api_key=\"...\")" + "agentops.init(api_key=\"...\")" ] }, { @@ -144,19 +149,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33magent\u001B[0m (to user):\n", + "\u001b[33magent\u001b[0m (to user):\n", "\n", "How can I help you today?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33muser\u001B[0m (to agent):\n", + "\u001b[33muser\u001b[0m (to agent):\n", "\n", "2+2\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33magent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33magent\u001b[0m (to user):\n", "\n", "2 + 2 equals 4.\n", "\n", @@ -168,7 +173,7 @@ "output_type": "stream", "text": [ "๐Ÿ–‡ AgentOps: This run's cost $0.000960\n", - "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001B[0m\u001B[0m\n" + "๐Ÿ–‡ AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001b[0m\u001b[0m\n" ] } ], @@ -185,7 +190,7 @@ "# Create the agent that represents the user in the conversation.\n", "user_proxy = UserProxyAgent(\"user\", code_execution_config=False)\n", "\n", - "# Let the assistant start the conversation. It will end when the user types exit.\n", + "# Let the assistant start the conversation. It will end when the user types \"exit\".\n", "assistant.initiate_chat(user_proxy, message=\"How can I help you today?\")\n", "\n", "# Close your AgentOps session to indicate that it completed.\n", @@ -204,13 +209,13 @@ }, { "cell_type": "markdown", - "source": [ - "![Session_Overview](https://github.com/AgentOps-AI/agentops/assets/14807319/d7228019-1488-40d3-852f-a61e998658ad)" - ], + "id": "cbd689b0f5617013", "metadata": { "collapsed": false }, - "id": "cbd689b0f5617013" + "source": [ + "![session replay](https://github.com/AgentOps-AI/agentops/blob/main/docs/images/external/app_screenshots/session-overview.png?raw=true)" + ] }, { "cell_type": "markdown", @@ -236,70 +241,70 @@ "name": "stderr", "output_type": "stream", "text": [ - "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001B[0m\u001B[0m\n" + "๐Ÿ–‡ AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001b[0m\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", "What is (1423 - 123) / 3 + (32 + 23) * 5?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", - "\u001B[32m***** Suggested tool call (call_aINcGyo0Xkrh9g7buRuhyCz0): calculator *****\u001B[0m\n", + "\u001b[32m***** Suggested tool call (call_aINcGyo0Xkrh9g7buRuhyCz0): calculator *****\u001b[0m\n", "Arguments: \n", "{\n", " \"a\": 1423,\n", " \"b\": 123,\n", " \"operator\": \"-\"\n", "}\n", - "\u001B[32m***************************************************************************\u001B[0m\n", + "\u001b[32m***************************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[32m***** Response from calling tool (call_aINcGyo0Xkrh9g7buRuhyCz0) *****\u001B[0m\n", + "\u001b[32m***** Response from calling tool (call_aINcGyo0Xkrh9g7buRuhyCz0) *****\u001b[0m\n", "1300\n", - "\u001B[32m**********************************************************************\u001B[0m\n", + "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", - "\u001B[32m***** Suggested tool call (call_prJGf8V0QVT7cbD91e0Fcxpb): calculator *****\u001B[0m\n", + "\u001b[32m***** Suggested tool call (call_prJGf8V0QVT7cbD91e0Fcxpb): calculator *****\u001b[0m\n", "Arguments: \n", "{\n", " \"a\": 1300,\n", " \"b\": 3,\n", " \"operator\": \"/\"\n", "}\n", - "\u001B[32m***************************************************************************\u001B[0m\n", + "\u001b[32m***************************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[32m***** Response from calling tool (call_prJGf8V0QVT7cbD91e0Fcxpb) *****\u001B[0m\n", + "\u001b[32m***** Response from calling tool (call_prJGf8V0QVT7cbD91e0Fcxpb) *****\u001b[0m\n", "433\n", - "\u001B[32m**********************************************************************\u001B[0m\n", + "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n" ] }, { @@ -316,94 +321,94 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", - "\u001B[32m***** Suggested tool call (call_CUIgHRsySLjayDKuUphI1TGm): calculator *****\u001B[0m\n", + "\u001b[32m***** Suggested tool call (call_CUIgHRsySLjayDKuUphI1TGm): calculator *****\u001b[0m\n", "Arguments: \n", "{\n", " \"a\": 32,\n", " \"b\": 23,\n", " \"operator\": \"+\"\n", "}\n", - "\u001B[32m***************************************************************************\u001B[0m\n", + "\u001b[32m***************************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[32m***** Response from calling tool (call_CUIgHRsySLjayDKuUphI1TGm) *****\u001B[0m\n", + "\u001b[32m***** Response from calling tool (call_CUIgHRsySLjayDKuUphI1TGm) *****\u001b[0m\n", "55\n", - "\u001B[32m**********************************************************************\u001B[0m\n", + "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", - "\u001B[32m***** Suggested tool call (call_L7pGtBLUf9V0MPL90BASyesr): calculator *****\u001B[0m\n", + "\u001b[32m***** Suggested tool call (call_L7pGtBLUf9V0MPL90BASyesr): calculator *****\u001b[0m\n", "Arguments: \n", "{\n", " \"a\": 55,\n", " \"b\": 5,\n", " \"operator\": \"*\"\n", "}\n", - "\u001B[32m***************************************************************************\u001B[0m\n", + "\u001b[32m***************************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[32m***** Response from calling tool (call_L7pGtBLUf9V0MPL90BASyesr) *****\u001B[0m\n", + "\u001b[32m***** Response from calling tool (call_L7pGtBLUf9V0MPL90BASyesr) *****\u001b[0m\n", "275\n", - "\u001B[32m**********************************************************************\u001B[0m\n", + "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", - "\u001B[32m***** Suggested tool call (call_Ygo6p4XfcxRjkYBflhG3UVv6): calculator *****\u001B[0m\n", + "\u001b[32m***** Suggested tool call (call_Ygo6p4XfcxRjkYBflhG3UVv6): calculator *****\u001b[0m\n", "Arguments: \n", "{\n", " \"a\": 433,\n", " \"b\": 275,\n", " \"operator\": \"+\"\n", "}\n", - "\u001B[32m***************************************************************************\u001B[0m\n", + "\u001b[32m***************************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[35m\n", - ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", - "\u001B[32m***** Response from calling tool (call_Ygo6p4XfcxRjkYBflhG3UVv6) *****\u001B[0m\n", + "\u001b[32m***** Response from calling tool (call_Ygo6p4XfcxRjkYBflhG3UVv6) *****\u001b[0m\n", "708\n", - "\u001B[32m**********************************************************************\u001B[0m\n", + "\u001b[32m**********************************************************************\u001b[0m\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", "The result of the calculation is 708.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser\u001B[0m (to Assistant):\n", + "\u001b[33mUser\u001b[0m (to Assistant):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mAssistant\u001B[0m (to User):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mAssistant\u001b[0m (to User):\n", "\n", "TERMINATE\n", "\n", @@ -415,7 +420,7 @@ "output_type": "stream", "text": [ "๐Ÿ–‡ AgentOps: This run's cost $0.001800\n", - "๐Ÿ–‡ AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001B[0m\u001B[0m\n" + "๐Ÿ–‡ AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001b[0m\u001b[0m\n" ] } ], @@ -474,7 +479,7 @@ " description=\"A simple calculator\", # A description of the tool.\n", ")\n", "\n", - "# Let the assistant start the conversation. It will end when the user types exit.\n", + "# Let the assistant start the conversation. It will end when the user types \"exit\".\n", "user_proxy.initiate_chat(assistant, message=\"What is (1423 - 123) / 3 + (32 + 23) * 5?\")\n", "\n", "agentops.end_session(\"Success\")" @@ -493,13 +498,13 @@ }, { "cell_type": "markdown", - "source": [ - "![Session Drilldown](https://github.com/AgentOps-AI/agentops/assets/14807319/561d59f3-c441-4066-914b-f6cfe32a598c)" - ], + "id": "a922a52ab5fce31", "metadata": { "collapsed": false }, - "id": "a922a52ab5fce31" + "source": [ + "![Session Drilldown](https://github.com/AgentOps-AI/agentops/blob/main/docs/images/external/app_screenshots/session-replay.png?raw=true)" + ] } ], "metadata": { diff --git a/website/docs/ecosystem/agentops.md b/website/docs/ecosystem/agentops.md index 76995b6eb5e4..581fb2671e97 100644 --- a/website/docs/ecosystem/agentops.md +++ b/website/docs/ecosystem/agentops.md @@ -1,29 +1,37 @@ -# AgentOps ๐Ÿ–‡๏ธ +# Agent Monitoring and Debugging with AgentOps -![logo](https://raw.githubusercontent.com/AgentOps-AI/agentops/35d5682866921a9e28d8ef66ae3c3b3d92d8fa6b/img/logo.png) +AgentOps logo -[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for agents. +[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for AI agents. At a high level, AgentOps gives you the ability to monitor LLM calls, costs, latency, agent failures, multi-agent interactions, tool usage, session-wide statistics, and more. For more info, check out the [AgentOps Repo](https://github.com/AgentOps-AI/agentops). +| | | +| ------------------------------------- | ------------------------------------------------------------- | +| ๐Ÿ“Š **Replay Analytics and Debugging** | Step-by-step agent execution graphs | +| ๐Ÿ’ธ **LLM Cost Management** | Track spend with LLM foundation model providers | +| ๐Ÿงช **Agent Benchmarking** | Test your agents against 1,000+ evals | +| ๐Ÿ” **Compliance and Security** | Detect common prompt injection and data exfiltration exploits | +| ๐Ÿค **Framework Integrations** | Native Integrations with CrewAI, AutoGen, & LangChain | +
- Agent Dashboard + Agent Dashboard - Agent Dashboard + Agent Dashboard
- Session Analytics + Session Analytics - Session Analytics + Session Analytics
- Session Replays + Session Replays - Session Replays + Session Replays
@@ -38,7 +46,7 @@ pip install agentops ``` 2. **Create an API Key:** -Create a user API key here: [Create API Key](https://app.agentops.ai/account) +Create a user API key here: [Create API Key](https://app.agentops.ai/settings/projects) 3. **Configure Your Environment:** Add your API key to your environment variables diff --git a/website/docs/topics/llm-observability.md b/website/docs/topics/llm-observability.md index 6a95d185f979..f80b55ea0982 100644 --- a/website/docs/topics/llm-observability.md +++ b/website/docs/topics/llm-observability.md @@ -1,42 +1,37 @@ -# LLM Observability +# Agent Observability -AutoGen supports advanced LLM observability and monitoring through built-in logging and partner providers. +AutoGen supports advanced LLM agent observability and monitoring through built-in logging and partner providers. -## What is LLM Observability -AI agent observability is the ability to monitor, measure, and understand the internal states and behaviors of AI agent systems. -Observability is crucial for ensuring transparency, reliability, and accountability in your agent systems. +## AutoGen Observability Integrations +### Built-In Logging +AutoGen's SQLite and File Logger - [Tutorial Notebook](/docs/notebooks/agentchat_logging) -## Development +### Full-Service Partner Integrations +AutoGen partners with [AgentOps](https://agentops.ai) to provide multi-agent tracking, metrics, and monitoring - [Tutorial Notebook](/docs/notebooks/agentchat_agentops) -### Agent Development in Terminal is Limited -- Lose track of what your agents did in between executions -- Parsing through terminal output searching for LLM completions -- Printing โ€œtool calledโ€ -### Agent Development Dashboards Enable More -- Visual dashboard so you can see what your agents did in human-readable format -- LLM calls are magically recorded - prompt, completion, timestamps for each - with one line of code -- Agents and their events (including tool calls) are recorded with one more line of code -- Errors are magically associated to its causal event -- Record any other events to your session with two more lines of code -- Tons of other useful data if youโ€™re developing with supported agent frameworks: SDK version +## What is Observability? +Observability provides developers with the necessary insights to understand and improve the internal workings of their agents. Observability is necessary for maintaining reliability, tracking costs, and ensuring AI safety. -## Compliance +**Without observability tools, developers face significant hurdles:** -Observability and monitoring is critical to ensure AI agent systems adhere to laws and regulations in industries like finance and healthcare, preventing violations such as data breaches and privacy issues. +- Tracking agent activities across sessions becomes a complex, error-prone task. +- Manually sifting through verbose terminal outputs to understand LLM interactions is inefficient. +- Pinpointing the exact moments of tool invocations is often like finding a needle in a haystack. -- Insights into AI decision-making, allowing organizations to explain outcomes and build trust with stakeholders. -- Helps detect anomalies and unintended behaviors early, mitigating operational, financial, and reputational risks. -- Ensures compliance with data privacy regulations, preventing unauthorized access and misuse of sensitive information. -- Quick identification and response to compliance violations, supporting incident analysis and prevention. -## Available Observability Integrations +**Key Features of Observability Dashboards:** +- Human-readable overview analytics and replays of agent activities. +- LLM cost, prompt, completion, timestamp, and metadata tracking for performance monitoring. +- Tool invocation, events, and agent-to-agent interactions for workflow monitoring. +- Error flagging and notifications for faster debugging. +- Access to a wealth of data for developers using supported agent frameworks, such as environments, SDK versions, and more. -### Logging -- Autogen SQLite and File Logger - [Tutorial](/docs/notebooks/agentchat_logging) +### Compliance -### Full-Service Partners -Autogen is currently partnered with [AgentOps](https://agentops.ai) for seamless observability integration. - -[Learn how to install AgentOps](/docs/notebooks/agentchat_agentops) +Observability is not just a development convenienceโ€”it's a compliance necessity, especially in regulated industries: +- It offers insights into AI decision-making processes, fostering trust and transparency. +- Anomalies and unintended behaviors are detected promptly, reducing various risks. +- Ensuring adherence to data privacy regulations, thereby safeguarding sensitive information. +- Compliance violations are quickly identified and addressed, enhancing incident management. From 03259b22290f05b120d881efe58daf1050f5af2b Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sat, 22 Jun 2024 09:31:52 -0400 Subject: [PATCH 03/57] bump version (#2999) Co-authored-by: Li Jiang --- autogen/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen/version.py b/autogen/version.py index 110d3e10d2f0..77fc1e2ea295 100644 --- a/autogen/version.py +++ b/autogen/version.py @@ -1 +1 @@ -__version__ = "0.2.30" +__version__ = "0.2.31" From 1582927ccd4de493be0d406ce59cb9cae729ed04 Mon Sep 17 00:00:00 2001 From: Yiran Wu <32823396+yiranwu0@users.noreply.github.com> Date: Sun, 23 Jun 2024 16:19:37 -0700 Subject: [PATCH 04/57] Improve doc in tutorial/conversation-patterns and customized_speaker_selection (#3006) * update * update --------- Co-authored-by: Yiran Wu <32823396+kevin666aa@users.noreply.github.com> --- website/blog/2023-06-28-MathChat/index.mdx | 2 +- .../blog/2023-11-13-OAI-assistants/index.mdx | 2 +- website/blog/authors.yml | 4 +- .../customized_speaker_selection.ipynb | 52 +++++++++++-------- .../docs/tutorial/conversation-patterns.ipynb | 13 ++++- 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/website/blog/2023-06-28-MathChat/index.mdx b/website/blog/2023-06-28-MathChat/index.mdx index 4c1007c611b8..be2423de9eed 100644 --- a/website/blog/2023-06-28-MathChat/index.mdx +++ b/website/blog/2023-06-28-MathChat/index.mdx @@ -75,7 +75,7 @@ We found that compared to basic prompting, which demonstrates the innate capabil For categories like Algebra and Prealgebra, PoT and PS showed little improvement, and in some instances, even led to a decrease in accuracy. However, MathChat was able to enhance total accuracy by around 6% compared to PoT and PS, showing competitive performance across all categories. Remarkably, MathChat improved accuracy in the Algebra category by about 15% over other methods. Note that categories like Intermediate Algebra and Precalculus remained challenging for all methods, with only about 20% of problems solved accurately. -The code for experiments can be found at this [repository](https://github.com/kevin666aa/FLAML/tree/gpt_math_solver/flaml/autogen/math). +The code for experiments can be found at this [repository](https://github.com/yiranwu0/FLAML/tree/gpt_math_solver/flaml/autogen/math). We now provide an implementation of MathChat using the interactive agents in AutoGen. See this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_MathChat.ipynb) for example usage. ## Future Directions diff --git a/website/blog/2023-11-13-OAI-assistants/index.mdx b/website/blog/2023-11-13-OAI-assistants/index.mdx index e73e31ad591f..07216a25969c 100644 --- a/website/blog/2023-11-13-OAI-assistants/index.mdx +++ b/website/blog/2023-11-13-OAI-assistants/index.mdx @@ -112,6 +112,6 @@ Checkout more examples [here](https://github.com/microsoft/autogen/tree/main/not `GPTAssistantAgent` was made possible through collaboration with [@IANTHEREAL](https://github.com/IANTHEREAL), [Jiale Liu](https://leoljl.github.io), -[Yiran Wu](https://github.com/kevin666aa), +[Yiran Wu](https://github.com/yiranwu0), [Qingyun Wu](https://qingyun-wu.github.io/), [Chi Wang](https://www.microsoft.com/en-us/research/people/chiw/), and many other AutoGen maintainers. diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 302bb8fceaaf..189853724ff6 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -13,8 +13,8 @@ qingyunwu: yiranwu: name: Yiran Wu title: PhD student at Pennsylvania State University - url: https://github.com/kevin666aa - image_url: https://github.com/kevin666aa.png + url: https://github.com/yiranwu0 + image_url: https://github.com/yiranwu0.png jialeliu: name: Jiale Liu diff --git a/website/docs/topics/groupchat/customized_speaker_selection.ipynb b/website/docs/topics/groupchat/customized_speaker_selection.ipynb index 830215a5e90b..2b800f3c8678 100644 --- a/website/docs/topics/groupchat/customized_speaker_selection.ipynb +++ b/website/docs/topics/groupchat/customized_speaker_selection.ipynb @@ -6,7 +6,34 @@ "source": [ "# Customize Speaker Selection\n", "\n", - "In GroupChat, we can also customize the speaker selection by passing in a function to `speaker_selection_method`:\n", + "```{=mdx}\n", + "![group_chat](../../../blog/2024-02-29-StateFlow/img/sf_example_1.png)\n", + "```\n", + "\n", + "In GroupChat, we can customize the speaker selection by passing a function to the `GroupChat` object. With this function, you can build a more **deterministic** agent workflow. We recommend following a **StateFlow** pattern when crafting this function. Please refer to the [StateFlow blog](/blog/2024/02/29/StateFlow) for more details.\n", + "\n", + "\n", + "## An example research workflow\n", + "We provide a simple example to build a StateFlow model for research with customized speaker selection.\n", + "\n", + "We first define the following agents:\n", + "\n", + "- Initializer: Start the workflow by sending a task.\n", + "- Coder: Retrieve papers from the internet by writing code.\n", + "- Executor: Execute the code.\n", + "- Scientist: Read the papers and write a summary.\n", + "\n", + "In the figure above, we define a simple workflow for research with 4 states: *Init*, *Retrieve*, *Research*, and *End*. Within each state, we will call different agents to perform the tasks.\n", + "\n", + "- *Init*: We use the initializer to start the workflow.\n", + "- *Retrieve*: We will first call the coder to write code and then call the executor to execute the code.\n", + "- *Research*: We will call the scientist to read the papers and write a summary.\n", + "- *End*: We will end the workflow.\n", + "\n", + "## Create your speaker selection function\n", + "\n", + "Below is a skeleton of the speaker selection function. Fill in the function to define the speaker selection logic.\n", + "\n", "```python\n", "def custom_speaker_selection_func(\n", " last_speaker: Agent, \n", @@ -35,28 +62,7 @@ ")\n", "```\n", "The last speaker and the groupchat object are passed to the function. \n", - "Commonly used variables from groupchat are `groupchat.messages` and `groupchat.agents`, which is the message history and the agents in the group chat respectively. You can access other attributes of the groupchat, such as `groupchat.allowed_speaker_transitions_dict` for pre-defined `allowed_speaker_transitions_dict`.\n", - "\n", - "Heres is a simple example to build workflow for research with customized speaker selection.\n", - "\n", - "\n", - "```{=mdx}\n", - "![group_chat](../../../blog/2024-02-29-StateFlow/img/sf_example_1.png)\n", - "```\n", - "\n", - "We define the following agents:\n", - "\n", - "- Initializer: Start the workflow by sending a task.\n", - "- Coder: Retrieve papers from the internet by writing code.\n", - "- Executor: Execute the code.\n", - "- Scientist: Read the papers and write a summary.\n", - "\n", - "In the Figure, we define a simple workflow for research with 4 states: Init, Retrieve, Research and End. Within each state, we will call different agents to perform the tasks.\n", - "\n", - "Init: We use the initializer to start the workflow.\n", - "Retrieve: We will first call the coder to write code and then call the executor to execute the code.\n", - "Research: We will call the scientist to read the papers and write a summary.\n", - "End: We will end the workflow." + "Commonly used variables from groupchat are `groupchat.messages` and `groupchat.agents`, which is the message history and the agents in the group chat respectively. You can access other attributes of the groupchat, such as `groupchat.allowed_speaker_transitions_dict` for pre-defined `allowed_speaker_transitions_dict`." ] }, { diff --git a/website/docs/tutorial/conversation-patterns.ipynb b/website/docs/tutorial/conversation-patterns.ipynb index eeaaa409b784..7ea8f0bfa517 100644 --- a/website/docs/tutorial/conversation-patterns.ipynb +++ b/website/docs/tutorial/conversation-patterns.ipynb @@ -12,7 +12,18 @@ "In this chapter, we will first dig a little bit more into the two-agent \n", "chat pattern and chat result, \n", "then we will show you several conversation patterns that involve \n", - "more than two agents.\n" + "more than two agents.\n", + "\n", + "### An Overview\n", + "\n", + "1. **Two-agent chat**: the simplest form of conversation pattern where two agents chat with each other.\n", + "2. **Sequential chat**: a sequence of chats between two agents, chained together by a carryover mechanism, which brings the summary of the previous chat to the context of the next chat.\n", + "3. **Group Chat**: a single chat involving more than two agents. An important question in group chat is: What agent should be next to speak? To support different scenarios, we provide different ways to organize agents in a group chat:\n", + " - We support several strategies to select the next agent: `round_robin`, `random`, `manual` (human selection), and `auto` (Default, using an LLM to decide).\n", + " - We provide a way to constrain the selection of the next speaker (See examples below).\n", + " - We allow you to pass in a function to customize the selection of the next speaker. With this feature, you can build a **StateFlow** model which allows a deterministic workflow among your agents.\n", + " Please refer to this [guide](/docs/topics/groupchat/customized_speaker_selection) and this [blog post](/blog/2024/02/29/StateFlow) on StateFlow for more details.\n", + "4. **Nested Chat**: package a workflow into a single agent for reuse in a larger workflow." ] }, { From 203954e9c154b3e37fc5483e0697adabbab6da2d Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Tue, 25 Jun 2024 09:23:24 -0700 Subject: [PATCH 05/57] [.Net] Update website with Tutorial section (#2982) * update * Update -> Releaes Notes * add ImageChat * update * update --- .../GettingStart/Image_Chat_With_Agent.cs | 13 ++- .../GettingStart/Use_Tools_With_Agent.cs | 60 ++++++---- dotnet/website/docfx.json | 2 + ...single-turn-tool-call-with-auto-invoke.png | 3 + ...gle-turn-tool-call-without-auto-invoke.png | 3 + dotnet/website/toc.yml | 5 +- dotnet/website/tutorial/Chat-with-an-agent.md | 53 +++++++++ .../tutorial/Create-agent-with-tools.md | 105 ++++++++++++++++++ .../website/tutorial/Image-chat-with-agent.md | 50 +++++++++ dotnet/website/tutorial/toc.yml | 8 ++ 10 files changed, 277 insertions(+), 25 deletions(-) create mode 100644 dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png create mode 100644 dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png create mode 100644 dotnet/website/tutorial/Chat-with-an-agent.md create mode 100644 dotnet/website/tutorial/Create-agent-with-tools.md create mode 100644 dotnet/website/tutorial/Image-chat-with-agent.md create mode 100644 dotnet/website/tutorial/toc.yml diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs index 3352f90d9211..5b94a238bbe8 100644 --- a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs @@ -1,10 +1,12 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Image_Chat_With_Agent.cs +#region Using using AutoGen.Core; using AutoGen.OpenAI; using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; +#endregion Using using FluentAssertions; namespace AutoGen.BasicSample; @@ -33,16 +35,17 @@ public static async Task RunAsync() var imageMessage = new ImageMessage(Role.User, BinaryData.FromBytes(imageBytes, "image/png")); #endregion Prepare_Image_Input - #region Chat_With_Agent - var reply = await agent.SendAsync("what's in the picture", chatHistory: [imageMessage]); - #endregion Chat_With_Agent - #region Prepare_Multimodal_Input var textMessage = new TextMessage(Role.User, "what's in the picture"); var multimodalMessage = new MultiModalMessage(Role.User, [textMessage, imageMessage]); - reply = await agent.SendAsync(multimodalMessage); #endregion Prepare_Multimodal_Input + #region Chat_With_Agent + var reply = await agent.SendAsync("what's in the picture", chatHistory: [imageMessage]); + // or use multimodal message to generate reply + reply = await agent.SendAsync(multimodalMessage); + #endregion Chat_With_Agent + #region verify_reply reply.Should().BeOfType(); #endregion verify_reply diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs index f1a230c123b1..b441fe389da2 100644 --- a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs @@ -11,6 +11,7 @@ namespace AutoGen.BasicSample; +#region Tools public partial class Tools { /// @@ -23,6 +24,8 @@ public async Task GetWeather(string city) return $"The weather in {city} is sunny."; } } +#endregion Tools + public class Use_Tools_With_Agent { public static async Task RunAsync() @@ -31,37 +34,53 @@ public static async Task RunAsync() var tools = new Tools(); #endregion Create_tools - #region Create_Agent - var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); - var model = "gpt-3.5-turbo"; - var openaiClient = new OpenAIClient(apiKey); - var functionCallMiddleware = new FunctionCallMiddleware( + #region Create_auto_invoke_middleware + var autoInvokeMiddleware = new FunctionCallMiddleware( functions: [tools.GetWeatherFunctionContract], functionMap: new Dictionary>>() { { tools.GetWeatherFunctionContract.Name!, tools.GetWeatherWrapper }, }); + #endregion Create_auto_invoke_middleware + + #region Create_no_invoke_middleware + var noInvokeMiddleware = new FunctionCallMiddleware( + functions: [tools.GetWeatherFunctionContract]); + #endregion Create_no_invoke_middleware + + #region Create_Agent + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); + var model = "gpt-3.5-turbo"; + var openaiClient = new OpenAIClient(apiKey); var agent = new OpenAIChatAgent( openAIClient: openaiClient, name: "agent", modelName: model, systemMessage: "You are a helpful AI assistant") - .RegisterMessageConnector() // convert OpenAI message to AutoGen message - .RegisterMiddleware(functionCallMiddleware) // pass function definition to agent. - .RegisterPrintMessage(); // print the message content + .RegisterMessageConnector(); // convert OpenAI message to AutoGen message #endregion Create_Agent - #region Single_Turn_Tool_Call + #region Single_Turn_Auto_Invoke + var autoInvokeAgent = agent + .RegisterMiddleware(autoInvokeMiddleware) // pass function definition to agent. + .RegisterPrintMessage(); // print the message content var question = new TextMessage(Role.User, "What is the weather in Seattle?"); - var toolCallReply = await agent.SendAsync(question); - #endregion Single_Turn_Tool_Call + var reply = await autoInvokeAgent.SendAsync(question); + reply.Should().BeOfType(); + #endregion Single_Turn_Auto_Invoke + + #region Single_Turn_No_Invoke + var noInvokeAgent = agent + .RegisterMiddleware(noInvokeMiddleware) // pass function definition to agent. + .RegisterPrintMessage(); // print the message content - #region verify_too_call_reply - toolCallReply.Should().BeOfType(); - #endregion verify_too_call_reply + question = new TextMessage(Role.User, "What is the weather in Seattle?"); + reply = await noInvokeAgent.SendAsync(question); + reply.Should().BeOfType(); + #endregion Single_Turn_No_Invoke #region Multi_Turn_Tool_Call - var finalReply = await agent.SendAsync(chatHistory: [question, toolCallReply]); + var finalReply = await agent.SendAsync(chatHistory: [question, reply]); #endregion Multi_Turn_Tool_Call #region verify_reply @@ -70,16 +89,19 @@ public static async Task RunAsync() #region parallel_tool_call question = new TextMessage(Role.User, "What is the weather in Seattle, New York and Vancouver"); - toolCallReply = await agent.SendAsync(question); + reply = await agent.SendAsync(question); #endregion parallel_tool_call #region verify_parallel_tool_call_reply - toolCallReply.Should().BeOfType(); - (toolCallReply as ToolCallAggregateMessage)!.Message1.ToolCalls.Count().Should().Be(3); + reply.Should().BeOfType(); + (reply as ToolCallAggregateMessage)!.Message1.ToolCalls.Count().Should().Be(3); #endregion verify_parallel_tool_call_reply #region Multi_Turn_Parallel_Tool_Call - finalReply = await agent.SendAsync(chatHistory: [question, toolCallReply]); + finalReply = await agent.SendAsync(chatHistory: [question, reply]); + finalReply.Should().BeOfType(); + (finalReply as ToolCallAggregateMessage)!.Message1.ToolCalls.Count().Should().Be(3); #endregion Multi_Turn_Parallel_Tool_Call } + } diff --git a/dotnet/website/docfx.json b/dotnet/website/docfx.json index e06f9797c1fb..224ef9065ca8 100644 --- a/dotnet/website/docfx.json +++ b/dotnet/website/docfx.json @@ -30,6 +30,8 @@ "files": [ "articles/**.md", "articles/**/toc.yml", + "tutorial/**.md", + "tutorial/**/toc.yml", "toc.yml", "*.md" ] diff --git a/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png new file mode 100644 index 000000000000..27914072b271 --- /dev/null +++ b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d8e2ab194e31dc70e39ba081a755c8e792d291bef4dc8b4c5cc372bed9ec50 +size 215389 diff --git a/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png new file mode 100644 index 000000000000..a0711e505e8c --- /dev/null +++ b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2e632fb24641eb2fac7fff995c9b3213023c45c3238531eec5a340072865f6 +size 202768 diff --git a/dotnet/website/toc.yml b/dotnet/website/toc.yml index 3931f5e79471..b92e92d2c363 100644 --- a/dotnet/website/toc.yml +++ b/dotnet/website/toc.yml @@ -1,10 +1,13 @@ - name: Docs href: articles/ + +- name: Tutorial + href: tutorial/ - name: API Reference href: api/ -- name: Update Log +- name: Release Notes href: update.md - name: Other Languages diff --git a/dotnet/website/tutorial/Chat-with-an-agent.md b/dotnet/website/tutorial/Chat-with-an-agent.md new file mode 100644 index 000000000000..11a73de341d4 --- /dev/null +++ b/dotnet/website/tutorial/Chat-with-an-agent.md @@ -0,0 +1,53 @@ +This tutorial shows how to generate response using an @AutoGen.Core.IAgent by taking @AutoGen.OpenAI.OpenAIChatAgent as an example. + +> [!NOTE] +> AutoGen.Net provides the following agents to connect to different LLM platforms. Generating responses using these agents is similar to the example shown below. +> - @AutoGen.OpenAI.OpenAIChatAgent +> - @AutoGen.SemanticKernel.SemanticKernelAgent +> - @AutoGen.LMStudio.LMStudioAgent +> - @AutoGen.Mistral.MistralClientAgent +> - @AutoGen.Anthropic.AnthropicClientAgent +> - @AutoGen.Ollama.OllamaAgent +> - @AutoGen.Gemini.GeminiChatAgent + +> [!NOTE] +> The complete code example can be found in [Chat_With_Agent.cs](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs) + +## Step 1: Install AutoGen + +First, install the AutoGen package using the following command: + +```bash +dotnet add package AutoGen +``` + +## Step 2: add Using Statements + +[!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Using)] + +## Step 3: Create an @AutoGen.OpenAI.OpenAIChatAgent + +> [!NOTE] +> The @AutoGen.OpenAI.Extension.OpenAIAgentExtension.RegisterMessageConnector* method registers an @AutoGen.OpenAI.OpenAIChatRequestMessageConnector middleware which converts OpenAI message types to AutoGen message types. This step is necessary when you want to use AutoGen built-in message types like @AutoGen.Core.TextMessage, @AutoGen.Core.ImageMessage, etc. +> For more information, see [Built-in-messages](../articles/Built-in-messages.md) + +[!code-csharp[Create an OpenAIChatAgent](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Create_Agent)] + +## Step 4: Generate Response +To generate response, you can use one of the overloaded method of @AutoGen.Core.AgentExtension.SendAsync* method. The following code shows how to generate response with text message: + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Chat_With_Agent)] + +To generate response with chat history, you can pass the chat history to the @AutoGen.Core.AgentExtension.SendAsync* method: + +[!code-csharp[Generate Response with Chat History](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Chat_With_History)] + +To streamingly generate response, use @AutoGen.Core.IStreamingAgent.GenerateStreamingReplyAsync* + +[!code-csharp[Generate Streaming Response](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Streaming_Chat)] + +## Further Reading +- [Chat with google gemini](../articles/AutoGen.Gemini/Chat-with-google-gemini.md) +- [Chat with vertex gemini](../articles/AutoGen.Gemini/Chat-with-vertex-gemini.md) +- [Chat with Ollama](../articles/AutoGen.Ollama/Chat-with-llama.md) +- [Chat with Semantic Kernel Agent](../articles/AutoGen.SemanticKernel/SemanticKernelAgent-simple-chat.md) \ No newline at end of file diff --git a/dotnet/website/tutorial/Create-agent-with-tools.md b/dotnet/website/tutorial/Create-agent-with-tools.md new file mode 100644 index 000000000000..02aade1b4860 --- /dev/null +++ b/dotnet/website/tutorial/Create-agent-with-tools.md @@ -0,0 +1,105 @@ +This tutorial shows how to use tools in an agent. + +## What is tool +Tools are pre-defined functions in user's project that agent can invoke. Agent can use tools to perform actions like search web, perform calculations, etc. With tools, it can greatly extend the capabilities of an agent. + +> [!NOTE] +> To use tools with agent, the backend LLM model used by the agent needs to support tool calling. Here are some of the LLM models that support tool calling as of 06/21/2024 +> - GPT-3.5-turbo with version >= 0613 +> - GPT-4 series +> - Gemini series +> - OPEN_MISTRAL_7B +> - ... +> +> This tutorial uses the latest `GPT-3.5-turbo` as example. + +> [!NOTE] +> The complete code example can be found in [Use_Tools_With_Agent.cs](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs) + +## Key Concepts +- @AutoGen.Core.FunctionContract: The contract of a function that agent can invoke. It contains the function name, description, parameters schema, and return type. +- @AutoGen.Core.ToolCallMessage: A message type that represents a tool call request in AutoGen.Net. +- @AutoGen.Core.ToolCallResultMessage: A message type that represents a tool call result in AutoGen.Net. +- @AutoGen.Core.ToolCallAggregateMessage: An aggregate message type that represents a tool call request and its result in a single message in AutoGen.Net. +- @AutoGen.Core.FunctionCallMiddleware: A middleware that pass the @AutoGen.Core.FunctionContract to the agent when generating response, and process the tool call response when receiving a @AutoGen.Core.ToolCallMessage. + +> [!Tip] +> You can Use AutoGen.SourceGenerator to automatically generate type-safe @AutoGen.Core.FunctionContract instead of manually defining them. For more information, please check out [Create type-safe function](../articles/Create-type-safe-function-call.md). + +## Install AutoGen and AutoGen.SourceGenerator +First, install the AutoGen and AutoGen.SourceGenerator package using the following command: + +```bash +dotnet add package AutoGen +dotnet add package AutoGen.SourceGenerator +``` + +Also, you might need to enable structural xml document support by setting `GenerateDocumentationFile` property to true in your project file. This allows source generator to leverage the documentation of the function when generating the function definition. + +```xml + + + true + +``` + +## Add Using Statements + +[!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Using)] + +## Define `Tool` class +Create a `public partial` class to host the tools you want to use in AutoGen agents. The method has to be a `public` instance method and its return type must be `Task`. After the methods is defined, mark them with @AutoGen.Core.FunctionAttribute attribute. + +In the following example, we define a `GetWeather` tool that returns the weather information of a city. + +[!code-csharp[Define Tool class](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Tools)] + +## Tool call without auto-invoke +In this case, when receiving a @AutoGen.Core.ToolCallMessage, the agent will not automatically invoke the tool. Instead, the agent will return the original message back to the user. The user can then decide whether to invoke the tool or not. + +![single-turn tool call without auto-invoke](../images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png) + +To implement this, you can create the @AutoGen.Core.FunctionCallMiddleware without passing the `functionMap` parameter to the constructor so that the middleware will not automatically invoke the tool once it receives a @AutoGen.Core.ToolCallMessage from its inner agent. + +[!code-csharp[Create_tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_tools)] + +[!code-csharp[Single-turn tool call without auto-invoke](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_no_invoke_middleware)] + +## Tool call with auto-invoke +In this case, the agent will automatically invoke the tool when receiving a @AutoGen.Core.ToolCallMessage and return the @AutoGen.Core.ToolCallAggregateMessage which contains both the tool call request and the tool call result. + +![single-turn tool call with auto-invoke](../images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png) + +To implement this, you can create the @AutoGen.Core.FunctionCallMiddleware with the `functionMap` parameter so that the middleware will automatically invoke the tool once it receives a @AutoGen.Core.ToolCallMessage from its inner agent. + +[!code-csharp[Single-turn tool call with auto-invoke](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_auto_invoke_middleware)] + +## Create an agent with the @AutoGen.Core.FunctionCallMiddleware and chat with the agent + +To use tools with an agent, you can create an agent and register the @AutoGen.Core.FunctionCallMiddleware to the agent that was just created above. + +[!code-csharp[Create an agent with tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_Agent)] + +To automatically invoke the tool, you can register the agent with `autoInvokeMiddleware`. In this situation, the agent will automatically invoke the tool when receiving a @AutoGen.Core.ToolCallMessage. And return the @AutoGen.Core.ToolCallAggregateMessage which contains both the tool call request and the tool call result. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_Auto_Invoke)] + +To retrieve the raw @AutoGen.Core.ToolCallMessage without invoking the tool, you can register the agent with `noInvokeMiddleware`. In this situation, the agent will return the original message back to the user when receiving a @AutoGen.Core.ToolCallMessage. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_No_Invoke)] + +## Send the tool call result back to LLM to generate further response +In some cases, you may want to send the tool call result back to the LLM to generate further response. To do this, you can send the tool call response from agent back to the LLM by calling the `SendAsync` method of the agent. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Multi_Turn_Tool_Call)] + +## Parallel tool call +Some LLM models support parallel tool call, which returns multiple tool calls in one single message. Note that @AutoGen.Core.FunctionCallMiddleware has already handled the parallel tool call for you. When it receives a @AutoGen.Core.ToolCallMessage that contains multiple tool calls, it will automatically invoke all the tools in the sequantial order and return the @AutoGen.Core.ToolCallAggregateMessage which contains all the tool call requests and results. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=parallel_tool_call)] + +## Further Reading +- [Function-call-with-openai](../articles/OpenAIChatAgent-use-function-call.md) +- [Function-call-with-gemini](../articles/AutoGen.Gemini/Function-call-with-gemini.md) +- [Use kernel plugin in other agents](../articles/AutoGen.SemanticKernel/Use-kernel-plugin-in-other-agents.md) +- [function call in mistral](../articles/MistralChatAgent-use-function-call.md) \ No newline at end of file diff --git a/dotnet/website/tutorial/Image-chat-with-agent.md b/dotnet/website/tutorial/Image-chat-with-agent.md new file mode 100644 index 000000000000..1e6d4b0ae2b4 --- /dev/null +++ b/dotnet/website/tutorial/Image-chat-with-agent.md @@ -0,0 +1,50 @@ +This tutorial shows how to perform image chat with an agent using the @AutoGen.OpenAI.OpenAIChatAgent as an example. + +> [!NOTE] +> To chat image with an agent, the model behind the agent needs to support image input. Here is a partial list of models that support image input: +> - gpt-4o +> - gemini-1.5 +> - llava +> - claude-3 +> - ... +> +> In this example, we are using the gpt-4o model as the backend model for the agent. + +> [!NOTE] +> The complete code example can be found in [Image_Chat_With_Agent.cs](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs) + +## Step 1: Install AutoGen + +First, install the AutoGen package using the following command: + +```bash +dotnet add package AutoGen +``` + +## Step 2: Add Using Statements + +[!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Using)] + +## Step 3: Create an @AutoGen.OpenAI.OpenAIChatAgent + +[!code-csharp[Create an OpenAIChatAgent](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Create_Agent)] + +## Step 4: Prepare Image Message + +In AutoGen, you can create an image message using either @AutoGen.Core.ImageMessage or @AutoGen.Core.MultiModalMessage. The @AutoGen.Core.ImageMessage takes a single image as input, whereas the @AutoGen.Core.MultiModalMessage allows you to pass multiple modalities like text or image. + +Here is how to create an image message using @AutoGen.Core.ImageMessage: +[!code-csharp[Create Image Message](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Prepare_Image_Input)] + +Here is how to create a multimodal message using @AutoGen.Core.MultiModalMessage: +[!code-csharp[Create MultiModal Message](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Prepare_Multimodal_Input)] + +## Step 5: Generate Response + +To generate response, you can use one of the overloaded methods of @AutoGen.Core.AgentExtension.SendAsync* method. The following code shows how to generate response with an image message: + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Chat_With_Agent)] + +## Further Reading +- [Image chat with gemini](../articles/AutoGen.Gemini/Image-chat-with-gemini.md) +- [Image chat with llava](../articles/AutoGen.Ollama/Chat-with-llava.md) \ No newline at end of file diff --git a/dotnet/website/tutorial/toc.yml b/dotnet/website/tutorial/toc.yml new file mode 100644 index 000000000000..f624ec4af289 --- /dev/null +++ b/dotnet/website/tutorial/toc.yml @@ -0,0 +1,8 @@ +- name: Chat with an agent + href: Chat-with-an-agent.md + +- name: Image chat with agent + href: Image-chat-with-agent.md + +- name: Create agent with tools + href: Create-agent-with-tools.md \ No newline at end of file From f438d980b061061282968262b5c1795c00d156ee Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Tue, 25 Jun 2024 09:52:55 -0700 Subject: [PATCH 06/57] fix #2975 (#3012) --- .../AutoGen.Core/Message/ToolCallMessage.cs | 13 +++++++++- .../OpenAIChatRequestMessageConnector.cs | 24 ++++++++++++------- .../OpenAIMessageTests.cs | 10 +++++--- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs index 396dba3d3a17..d0f89e1ecdde 100644 --- a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs @@ -36,7 +36,7 @@ public override string ToString() } } -public class ToolCallMessage : IMessage, ICanGetToolCalls +public class ToolCallMessage : IMessage, ICanGetToolCalls, ICanGetTextContent { public ToolCallMessage(IEnumerable toolCalls, string? from = null) { @@ -80,6 +80,12 @@ public void Update(ToolCallMessageUpdate update) public string? From { get; set; } + /// + /// Some LLMs might also include text content in a tool call response, like GPT. + /// This field is used to store the text content in that case. + /// + public string? Content { get; set; } + public override string ToString() { var sb = new StringBuilder(); @@ -96,6 +102,11 @@ public IEnumerable GetToolCalls() { return this.ToolCalls; } + + public string? GetContent() + { + return this.Content; + } } public class ToolCallMessageUpdate : IStreamingMessage diff --git a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs index 246e50cc6c59..c1dc2caa99fb 100644 --- a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs +++ b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs @@ -136,14 +136,13 @@ private IMessage PostProcessChatCompletions(IMessage message) private IMessage PostProcessChatResponseMessage(ChatResponseMessage chatResponseMessage, string? from) { - if (chatResponseMessage.Content is string content && !string.IsNullOrEmpty(content)) - { - return new TextMessage(Role.Assistant, content, from); - } - + var textContent = chatResponseMessage.Content; if (chatResponseMessage.FunctionCall is FunctionCall functionCall) { - return new ToolCallMessage(functionCall.Name, functionCall.Arguments, from); + return new ToolCallMessage(functionCall.Name, functionCall.Arguments, from) + { + Content = textContent, + }; } if (chatResponseMessage.ToolCalls.Where(tc => tc is ChatCompletionsFunctionToolCall).Any()) @@ -154,7 +153,15 @@ private IMessage PostProcessChatResponseMessage(ChatResponseMessage chatResponse var toolCalls = functionToolCalls.Select(tc => new ToolCall(tc.Name, tc.Arguments) { ToolCallId = tc.Id }); - return new ToolCallMessage(toolCalls, from); + return new ToolCallMessage(toolCalls, from) + { + Content = textContent, + }; + } + + if (textContent is string content && !string.IsNullOrEmpty(content)) + { + return new TextMessage(Role.Assistant, content, from); } throw new InvalidOperationException("Invalid ChatResponseMessage"); @@ -327,7 +334,8 @@ private IEnumerable ProcessToolCallMessage(IAgent agent, Too } var toolCall = message.ToolCalls.Select((tc, i) => new ChatCompletionsFunctionToolCall(tc.ToolCallId ?? $"{tc.FunctionName}_{i}", tc.FunctionName, tc.FunctionArguments)); - var chatRequestMessage = new ChatRequestAssistantMessage(string.Empty) { Name = message.From }; + var textContent = message.GetContent() ?? string.Empty; + var chatRequestMessage = new ChatRequestAssistantMessage(textContent) { Name = message.From }; foreach (var tc in toolCall) { chatRequestMessage.ToolCalls.Add(tc); diff --git a/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs b/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs index 81581d068ee7..a9b852e0d8c1 100644 --- a/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs +++ b/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs @@ -278,9 +278,9 @@ public async Task ItProcessToolCallMessageAsync() var innerMessage = msgs.Last(); innerMessage!.Should().BeOfType>(); var chatRequestMessage = (ChatRequestAssistantMessage)((MessageEnvelope)innerMessage!).Content; - chatRequestMessage.Content.Should().BeNullOrEmpty(); chatRequestMessage.Name.Should().Be("assistant"); chatRequestMessage.ToolCalls.Count().Should().Be(1); + chatRequestMessage.Content.Should().Be("textContent"); chatRequestMessage.ToolCalls.First().Should().BeOfType(); var functionToolCall = (ChatCompletionsFunctionToolCall)chatRequestMessage.ToolCalls.First(); functionToolCall.Name.Should().Be("test"); @@ -291,7 +291,10 @@ public async Task ItProcessToolCallMessageAsync() .RegisterMiddleware(middleware); // user message - IMessage message = new ToolCallMessage("test", "test", "assistant"); + IMessage message = new ToolCallMessage("test", "test", "assistant") + { + Content = "textContent", + }; await agent.GenerateReplyAsync([message]); } @@ -526,13 +529,14 @@ public async Task ItConvertChatResponseMessageToToolCallMessageAsync() .RegisterMiddleware(middleware); // tool call message - var toolCallMessage = CreateInstance(ChatRole.Assistant, "", new[] { new ChatCompletionsFunctionToolCall("test", "test", "test") }, new FunctionCall("test", "test"), CreateInstance(), new Dictionary()); + var toolCallMessage = CreateInstance(ChatRole.Assistant, "textContent", new[] { new ChatCompletionsFunctionToolCall("test", "test", "test") }, new FunctionCall("test", "test"), CreateInstance(), new Dictionary()); var chatRequestMessage = MessageEnvelope.Create(toolCallMessage); var message = await agent.GenerateReplyAsync([chatRequestMessage]); message.Should().BeOfType(); message.GetToolCalls()!.Count().Should().Be(1); message.GetToolCalls()!.First().FunctionName.Should().Be("test"); message.GetToolCalls()!.First().FunctionArguments.Should().Be("test"); + message.GetContent().Should().Be("textContent"); } [Fact] From 17dbaa514eef93dc1003260bf3752c73ae451af4 Mon Sep 17 00:00:00 2001 From: James Woffinden-Luey <90423712+jluey1@users.noreply.github.com> Date: Tue, 25 Jun 2024 11:02:52 -0700 Subject: [PATCH 07/57] AgentEval Blogpost (#2954) * first draft of agent eval blog post * adding NextSteps section * Update website/blog/2024-06-21-AgentEval/index.mdx Co-authored-by: Chi Wang * Update website/blog/2024-06-21-AgentEval/index.mdx Co-authored-by: Chi Wang * addressing some pr comments * fixing whitespace * fixing typo * adding bit about sequential chats * fixing whitespace * adding more about verifier --------- Co-authored-by: Beibin Li Co-authored-by: Chi Wang --- .../img/agenteval_ov_v3.png | 3 + website/blog/2024-06-21-AgentEval/index.mdx | 202 ++++++++++++++++++ website/blog/authors.yml | 5 + 3 files changed, 210 insertions(+) create mode 100644 website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png create mode 100644 website/blog/2024-06-21-AgentEval/index.mdx diff --git a/website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png b/website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png new file mode 100644 index 000000000000..fe31283d72d0 --- /dev/null +++ b/website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e17aebbc38a8ba55e4b18b9352a680edcca4b3d6625b16f6d1ab3131da799b63 +size 236624 diff --git a/website/blog/2024-06-21-AgentEval/index.mdx b/website/blog/2024-06-21-AgentEval/index.mdx new file mode 100644 index 000000000000..87ffc857e839 --- /dev/null +++ b/website/blog/2024-06-21-AgentEval/index.mdx @@ -0,0 +1,202 @@ +--- +title: "AgentEval: A Developer Tool to Assess Utility of LLM-powered Applications" +authors: + - jluey + - julianakiseleva +tags: [LLM, GPT, evaluation, task utility] +--- + + +![Fig.1: An AgentEval framework with verification step](img/agenteval_ov_v3.png) + +

Fig.1 illustrates the general flow of AgentEval with verification step

+ + + +TL;DR: +* As a developer, how can you assess the utility and effectiveness of an LLM-powered application in helping end users with their tasks? +* To shed light on the question above, we previously introduced [`AgentEval`](https://microsoft.github.io/autogen/blog/2023/11/20/AgentEval/) โ€” a framework to assess the multi-dimensional utility of any LLM-powered application crafted to assist users in specific tasks. We have now embedded it as part of the AutoGen library to ease developer adoption. +* Here, we introduce an updated version of AgentEval that includes a verification process to estimate the robustness of the QuantifierAgent. More details can be found in [this paper](https://arxiv.org/abs/2405.02178). + + +## Introduction + +Previously introduced [`AgentEval`](https://microsoft.github.io/autogen/blog/2023/11/20/AgentEval/) is a comprehensive framework designed to bridge the gap in assessing the utility of LLM-powered applications. It leverages recent advancements in LLMs to offer a scalable and cost-effective alternative to traditional human evaluations. The framework comprises three main agents: `CriticAgent`, `QuantifierAgent`, and `VerifierAgent`, each playing a crucial role in assessing the task utility of an application. + +**CriticAgent: Defining the Criteria** + +The CriticAgent's primary function is to suggest a set of criteria for evaluating an application based on the task description and examples of successful and failed executions. For instance, in the context of a math tutoring application, the CriticAgent might propose criteria such as efficiency, clarity, and correctness. These criteria are essential for understanding the various dimensions of the application's performance. Itโ€™s highly recommended that application developers validate the suggested criteria leveraging their domain expertise. + +**QuantifierAgent: Quantifying the Performance** + +Once the criteria are established, the QuantifierAgent takes over to quantify how well the application performs against each criterion. This quantification process results in a multi-dimensional assessment of the application's utility, providing a detailed view of its strengths and weaknesses. + +**VerifierAgent: Ensuring Robustness and Relevance** + +VerifierAgent ensures the criteria used to evaluate a utility are effective for the end-user, maintaining both robustness and high discriminative power. It does this through two main actions: + +1. Criteria Stability: + * Ensures criteria are essential, non-redundant, and consistently measurable. + * Iterates over generating and quantifying criteria, eliminating redundancies, and evaluating their stability. + * Retains only the most robust criteria. + +2. Discriminative Power: + + * Tests the system's reliability by introducing adversarial examples (noisy or compromised data). + * Assesses the system's ability to distinguish these from standard cases. + * If the system fails, it indicates the need for better criteria to handle varied conditions effectively. + +## A Flexible and Scalable Framework + +One of AgentEval's key strengths is its flexibility. It can be applied to a wide range of tasks where success may or may not be clearly defined. For tasks with well-defined success criteria, such as household chores, the framework can evaluate whether multiple successful solutions exist and how they compare. For more open-ended tasks, such as generating an email template, AgentEval can assess the utility of the system's suggestions. + +Furthermore, AgentEval allows for the incorporation of human expertise. Domain experts can participate in the evaluation process by suggesting relevant criteria or verifying the usefulness of the criteria identified by the agents. This human-in-the-loop approach ensures that the evaluation remains grounded in practical, real-world considerations. + +## Empirical Validation + +To validate AgentEval, the framework was tested on two applications: math problem solving and ALFWorld, a household task simulation. The math dataset comprised 12,500 challenging problems, each with step-by-step solutions, while the ALFWorld dataset involved multi-turn interactions in a simulated environment. In both cases, AgentEval successfully identified relevant criteria, quantified performance, and verified the robustness of the evaluations, demonstrating its effectiveness and versatility. + +## How to use `AgentEval` + +AgentEval currently has two main stages; criteria generation and criteria quantification (criteria verification is still under development). Both stages make use of sequential LLM-powered agents to make their determinations. + +**Criteria Generation:** + +During criteria generation, AgentEval uses example execution message chains to create a set of criteria for quantifying how well an application performed for a given task. + +``` +def generate_criteria( + llm_config: Optional[Union[Dict, Literal[False]]] = None, + task: Task = None, + additional_instructions: str = "", + max_round=2, + use_subcritic: bool = False, +) +``` + +Parameters: +* llm_config (dict or bool): llm inference configuration. +* task ([Task](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval/task.py)): The task to evaluate. +* additional_instructions (str, optional): Additional instructions for the criteria agent. +* max_round (int, optional): The maximum number of rounds to run the conversation. +* use_subcritic (bool, optional): Whether to use the Subcritic agent to generate subcriteria. The Subcritic agent will break down a generated criteria into smaller criteria to be assessed. + +Example code: +``` +config_list = autogen.config_list_from_json("OAI_CONFIG_LIST") +task = Task( + **{ + "name": "Math problem solving", + "description": "Given any question, the system needs to solve the problem as consisely and accurately as possible", + "successful_response": response_successful, + "failed_response": response_failed, + } +) + +criteria = generate_criteria(task=task, llm_config={"config_list": config_list}) +``` + +Note: Only one sample execution chain (success/failure) is required for the task object but AgentEval will perform better with an example for each case. + + +Example Output: +``` +[ + { + "name": "Accuracy", + "description": "The solution must be correct and adhere strictly to mathematical principles and techniques appropriate for the problem.", + "accepted_values": ["Correct", "Minor errors", "Major errors", "Incorrect"] + }, + { + "name": "Conciseness", + "description": "The explanation and method provided should be direct and to the point, avoiding unnecessary steps or complexity.", + "accepted_values": ["Very concise", "Concise", "Somewhat verbose", "Verbose"] + }, + { + "name": "Relevance", + "description": "The content of the response must be relevant to the question posed and should address the specific problem requirements.", + "accepted_values": ["Highly relevant", "Relevant", "Somewhat relevant", "Not relevant"] + } +] +``` + + + +**Criteria Quantification:** + +During the quantification stage, AgentEval will use the generated criteria (or user defined criteria) to assess a given execution chain to determine how well the application performed. + +``` +def quantify_criteria( + llm_config: Optional[Union[Dict, Literal[False]]], + criteria: List[Criterion], + task: Task, + test_case: str, + ground_truth: str, +) +``` + +Parameters: +* llm_config (dict or bool): llm inference configuration. +* criteria ([Criterion](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval/criterion.py)): A list of criteria for evaluating the utility of a given task. This can either be generated by the `generate_criteria` function or manually created. +* task ([Task](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval/task.py)): The task to evaluate. It should match the one used during the `generate_criteria` step. +* test_case (str): The execution chain to assess. Typically this is a json list of messages but could be any string representation of a conversation chain. +* ground_truth (str): The ground truth for the test case. + +Example Code: +``` +test_case="""[ + { + "content": "Find $24^{-1} \\pmod{11^2}$. That is, find the residue $b$ for which $24b \\equiv 1\\pmod{11^2}$.\n\nExpress your answer as an integer from $0$ to $11^2-1$, inclusive.", + "role": "user" + }, + { + "content": "To find the modular inverse of 24 modulo 11^2, we can use the Extended Euclidean Algorithm. Here is a Python function to compute the modular inverse using this algorithm:\n\n```python\ndef mod_inverse(a, m):\n..." + "role": "assistant" + } + ]""" + +quantifier_output = quantify_criteria( + llm_config={"config_list": config_list}, + criteria=criteria, + task=task, + test_case=test_case, + ground_truth="true", +) +``` + +The output will be a json object consisting of the ground truth and a dictionary mapping each criteria to it's score. + +``` +{ + "actual_success": true, + "estimated_performance": { + "Accuracy": "Correct", + "Conciseness": "Concise", + "Relevance": "Highly relevant" + } +} +``` + +## What is next? +* Enabling AgentEval in AutoGen Studio for a nocode solution. +* Fully implementing VerifierAgent in the AgentEval framework. + +## Conclusion + +AgentEval represents a significant advancement in the evaluation of LLM-powered applications. By combining the strengths of CriticAgent, QuantifierAgent, and VerifierAgent, the framework offers a robust, scalable, and flexible solution for assessing task utility. This innovative approach not only helps developers understand the current performance of their applications but also provides valuable insights that can drive future improvements. As the field of intelligent agents continues to evolve, frameworks like AgentEval will play a crucial role in ensuring that these applications meet the diverse and dynamic needs of their users. + + +## Further reading + +Please refer to our [paper](https://arxiv.org/abs/2405.02178) and [codebase](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval) for more details about AgentEval. + +If you find this blog useful, please consider citing: +```bobtex +@article{arabzadeh2024assessing, + title={Assessing and Verifying Task Utility in LLM-Powered Applications}, + author={Arabzadeh, Negar and Huo, Siging and Mehta, Nikhil and Wu, Qinqyun and Wang, Chi and Awadallah, Ahmed and Clarke, Charles LA and Kiseleva, Julia}, + journal={arXiv preprint arXiv:2405.02178}, + year={2024} +} +``` diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 189853724ff6..83ea099ade48 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -123,3 +123,8 @@ yifanzeng: title: PhD student at Oregon State University url: https://xhmy.github.io/ image_url: https://xhmy.github.io/assets/img/photo.JPG + +jluey: + name: James Woffinden-Luey + title: Senior Research Engineer at Microsoft Research + url: https://github.com/jluey1 From 5e744f7fe4110421e03262dbf457f50d59eb83f8 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Tue, 25 Jun 2024 14:40:52 -0700 Subject: [PATCH 08/57] improve `Create agent with tools` and add tuturial reference in index.md (#3024) --- dotnet/website/articles/getting-start.md | 2 ++ dotnet/website/articles/toc.yml | 2 ++ dotnet/website/index.md | 3 --- .../tutorial/Create-agent-with-tools.md | 27 +++++++++---------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dotnet/website/articles/getting-start.md b/dotnet/website/articles/getting-start.md index 53cc7c9758f3..9db8494ff154 100644 --- a/dotnet/website/articles/getting-start.md +++ b/dotnet/website/articles/getting-start.md @@ -17,6 +17,8 @@ Then you can start with the following code snippet to create a conversable agent [!code-csharp[](../../sample/AutoGen.BasicSamples/CodeSnippet/GetStartCodeSnippet.cs?name=snippet_GetStartCodeSnippet)] [!code-csharp[](../../sample/AutoGen.BasicSamples/CodeSnippet/GetStartCodeSnippet.cs?name=code_snippet_1)] +### Tutorial +Getting started with AutoGen.Net by following the [tutorial](../tutorial/Chat-with-an-agent.md) series. ### Examples You can find more examples under the [sample project](https://github.com/microsoft/autogen/tree/dotnet/dotnet/sample/AutoGen.BasicSamples). diff --git a/dotnet/website/articles/toc.yml b/dotnet/website/articles/toc.yml index 837ecd6f86e0..6d3787ae3d0b 100644 --- a/dotnet/website/articles/toc.yml +++ b/dotnet/website/articles/toc.yml @@ -1,5 +1,7 @@ - name: Getting start items: + - name: Overview + href: ../index.md - name: Installation href: Installation.md - name: agent diff --git a/dotnet/website/index.md b/dotnet/website/index.md index 3bc691523e9b..164e5c1cf81c 100644 --- a/dotnet/website/index.md +++ b/dotnet/website/index.md @@ -1,4 +1 @@ ---- -_disableTocFilter: true ---- [!INCLUDE [](./articles/getting-start.md)] \ No newline at end of file diff --git a/dotnet/website/tutorial/Create-agent-with-tools.md b/dotnet/website/tutorial/Create-agent-with-tools.md index 02aade1b4860..d3af7bff6f67 100644 --- a/dotnet/website/tutorial/Create-agent-with-tools.md +++ b/dotnet/website/tutorial/Create-agent-with-tools.md @@ -47,12 +47,19 @@ Also, you might need to enable structural xml document support by setting `Gener [!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Using)] -## Define `Tool` class +## Create agent + +Create an @AutoGen.OpenAI.OpenAIChatAgent with `GPT-3.5-turbo` as the backend LLM model. + +[!code-csharp[Create an agent with tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_Agent)] + +## Define `Tool` class and create tools Create a `public partial` class to host the tools you want to use in AutoGen agents. The method has to be a `public` instance method and its return type must be `Task`. After the methods is defined, mark them with @AutoGen.Core.FunctionAttribute attribute. In the following example, we define a `GetWeather` tool that returns the weather information of a city. [!code-csharp[Define Tool class](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Tools)] +[!code-csharp[Create tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_tools)] ## Tool call without auto-invoke In this case, when receiving a @AutoGen.Core.ToolCallMessage, the agent will not automatically invoke the tool. Instead, the agent will return the original message back to the user. The user can then decide whether to invoke the tool or not. @@ -61,10 +68,12 @@ In this case, when receiving a @AutoGen.Core.ToolCallMessage, the agent will not To implement this, you can create the @AutoGen.Core.FunctionCallMiddleware without passing the `functionMap` parameter to the constructor so that the middleware will not automatically invoke the tool once it receives a @AutoGen.Core.ToolCallMessage from its inner agent. -[!code-csharp[Create_tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_tools)] - [!code-csharp[Single-turn tool call without auto-invoke](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_no_invoke_middleware)] +After creating the function call middleware, you can register it to the agent using `RegisterMiddleware` method, which will return a new agent which can use the methods defined in the `Tool` class. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_No_Invoke)] + ## Tool call with auto-invoke In this case, the agent will automatically invoke the tool when receiving a @AutoGen.Core.ToolCallMessage and return the @AutoGen.Core.ToolCallAggregateMessage which contains both the tool call request and the tool call result. @@ -74,20 +83,10 @@ To implement this, you can create the @AutoGen.Core.FunctionCallMiddleware with [!code-csharp[Single-turn tool call with auto-invoke](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_auto_invoke_middleware)] -## Create an agent with the @AutoGen.Core.FunctionCallMiddleware and chat with the agent - -To use tools with an agent, you can create an agent and register the @AutoGen.Core.FunctionCallMiddleware to the agent that was just created above. - -[!code-csharp[Create an agent with tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_Agent)] - -To automatically invoke the tool, you can register the agent with `autoInvokeMiddleware`. In this situation, the agent will automatically invoke the tool when receiving a @AutoGen.Core.ToolCallMessage. And return the @AutoGen.Core.ToolCallAggregateMessage which contains both the tool call request and the tool call result. +After creating the function call middleware, you can register it to the agent using `RegisterMiddleware` method, which will return a new agent which can use the methods defined in the `Tool` class. [!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_Auto_Invoke)] -To retrieve the raw @AutoGen.Core.ToolCallMessage without invoking the tool, you can register the agent with `noInvokeMiddleware`. In this situation, the agent will return the original message back to the user when receiving a @AutoGen.Core.ToolCallMessage. - -[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_No_Invoke)] - ## Send the tool call result back to LLM to generate further response In some cases, you may want to send the tool call result back to the LLM to generate further response. To do this, you can send the tool call response from agent back to the LLM by calling the `SendAsync` method of the agent. From 08726421fc2db7d5ec00d1dc99a5aa1af2a101a4 Mon Sep 17 00:00:00 2001 From: Jeffrey Su Date: Wed, 26 Jun 2024 07:09:08 +0800 Subject: [PATCH 09/57] #2708 add Add a judgment to the graph constructor (#2709) * #2708 add Add a judgment to the graph constructor * #2708 add Add a judgment to the graph constructor & added unit test * #2708 #2079 move GraphTests to AutoGen.Tests; delete AutoGen.Core.Tests project --- dotnet/AutoGen.sln | 4 ++-- dotnet/src/AutoGen.Core/GroupChat/Graph.cs | 11 +++++++++-- .../test/AutoGen.Tests/GroupChat/GraphTests.cs | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs diff --git a/dotnet/AutoGen.sln b/dotnet/AutoGen.sln index 5ecfe1938873..cd1d683406cd 100644 --- a/dotnet/AutoGen.sln +++ b/dotnet/AutoGen.sln @@ -1,4 +1,4 @@ -๏ปฟ + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34322.80 @@ -206,4 +206,4 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B} EndGlobalSection -EndGlobal +EndGlobal \ No newline at end of file diff --git a/dotnet/src/AutoGen.Core/GroupChat/Graph.cs b/dotnet/src/AutoGen.Core/GroupChat/Graph.cs index 02f4da50bae0..d6b71e2a3f13 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/Graph.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/Graph.cs @@ -12,9 +12,16 @@ public class Graph { private readonly List transitions = new List(); - public Graph(IEnumerable transitions) + public Graph() { - this.transitions.AddRange(transitions); + } + + public Graph(IEnumerable? transitions) + { + if (transitions != null) + { + this.transitions.AddRange(transitions); + } } public void AddTransition(Transition transition) diff --git a/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs b/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs new file mode 100644 index 000000000000..77e2c99dcd13 --- /dev/null +++ b/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs @@ -0,0 +1,18 @@ +๏ปฟ +using Xunit; + +namespace AutoGen.Tests +{ + public class GraphTests + { + [Fact] + public void GraphTest() + { + var graph1 = new Graph(); + Assert.NotNull(graph1); + + var graph2 = new Graph(null); + Assert.NotNull(graph2); + } + } +} From 57ec13c2eb1fd227a7976c62d0fd4a88bf8a1975 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 26 Jun 2024 08:33:28 -0700 Subject: [PATCH 10/57] [.Net] add sample on how to make function call using lite llm and ollama Plus move ollama openai sample to AutoGen.OpenAI.Sample project (#3015) * add sample * Update Connect_To_Ollama.cs * Update Connect_To_Ollama.cs --- dotnet/AutoGen.sln | 7 +++ .../AutoGen.OpenAI.Sample.csproj | 21 +++++++ .../Connect_To_Ollama.cs | 61 +++++++++++++++++++ .../sample/AutoGen.OpenAI.Sample/Program.cs | 6 ++ .../Tool_Call_With_Ollama_And_LiteLLM.cs | 55 +++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj create mode 100644 dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs create mode 100644 dotnet/sample/AutoGen.OpenAI.Sample/Program.cs create mode 100644 dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs diff --git a/dotnet/AutoGen.sln b/dotnet/AutoGen.sln index cd1d683406cd..5fa215f0ce9c 100644 --- a/dotnet/AutoGen.sln +++ b/dotnet/AutoGen.sln @@ -61,6 +61,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Gemini.Sample", "sa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.AotCompatibility.Tests", "test\AutoGen.AotCompatibility.Tests\AutoGen.AotCompatibility.Tests.csproj", "{6B82F26D-5040-4453-B21B-C8D1F913CE4C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.OpenAI.Sample", "sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj", "{0E635268-351C-4A6B-A28D-593D868C2CA4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -171,6 +173,10 @@ Global {6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU {6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU {6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Release|Any CPU.Build.0 = Release|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -202,6 +208,7 @@ Global {8EA16BAB-465A-4C07-ABC4-1070D40067E9} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {19679B75-CE3A-4DF0-A3F0-CA369D2760A4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} {6B82F26D-5040-4453-B21B-C8D1F913CE4C} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} + {0E635268-351C-4A6B-A28D-593D868C2CA4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B} diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj b/dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj new file mode 100644 index 000000000000..ffe18f8a616a --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj @@ -0,0 +1,21 @@ +๏ปฟ + + + Exe + net8.0 + enable + enable + True + $(NoWarn);CS8981;CS8600;CS8602;CS8604;CS8618;CS0219;SKEXP0054;SKEXP0050;SKEXP0110 + true + + + + + + + + + + + diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs new file mode 100644 index 000000000000..b4206b4b6c22 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs @@ -0,0 +1,61 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs +#region using_statement +using AutoGen.Core; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; +#endregion using_statement + +namespace AutoGen.OpenAI.Sample; + +#region CustomHttpClientHandler +public sealed class CustomHttpClientHandler : HttpClientHandler +{ + private string _modelServiceUrl; + + public CustomHttpClientHandler(string modelServiceUrl) + { + _modelServiceUrl = modelServiceUrl; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.RequestUri = new Uri($"{_modelServiceUrl}{request.RequestUri.PathAndQuery}"); + + return base.SendAsync(request, cancellationToken); + } +} +#endregion CustomHttpClientHandler + +public class Connect_To_Ollama +{ + public static async Task RunAsync() + { + #region create_agent + using var client = new HttpClient(new CustomHttpClientHandler("http://localhost:11434")); + var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) + { + Transport = new HttpClientTransport(client), + }; + + // api-key is not required for local server + // so you can use any string here + var openAIClient = new OpenAIClient("api-key", option); + var model = "llama3"; + + var agent = new OpenAIChatAgent( + openAIClient: openAIClient, + name: "assistant", + modelName: model, + systemMessage: "You are a helpful assistant designed to output JSON.", + seed: 0) + .RegisterMessageConnector() + .RegisterPrintMessage(); + #endregion create_agent + + #region send_message + await agent.SendAsync("Can you write a piece of C# code to calculate 100th of fibonacci?"); + #endregion send_message + } +} diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Program.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Program.cs new file mode 100644 index 000000000000..5a38a3ff03b9 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Program.cs @@ -0,0 +1,6 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Program.cs + +using AutoGen.OpenAI.Sample; + +Tool_Call_With_Ollama_And_LiteLLM.RunAsync().Wait(); diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs new file mode 100644 index 000000000000..f4fabe3c9e83 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs @@ -0,0 +1,55 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Tool_Call_With_Ollama_And_LiteLLM.cs + +using AutoGen.Core; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; + +namespace AutoGen.OpenAI.Sample; + +public partial class Function +{ + [Function] + public async Task GetWeatherAsync(string city) + { + return await Task.FromResult("The weather in " + city + " is 72 degrees and sunny."); + } +} +public class Tool_Call_With_Ollama_And_LiteLLM +{ + public static async Task RunAsync() + { + #region Create_Agent + var liteLLMUrl = "http://localhost:4000"; + using var httpClient = new HttpClient(new CustomHttpClientHandler(liteLLMUrl)); + var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) + { + Transport = new HttpClientTransport(httpClient), + }; + + var functions = new Function(); + var functionMiddleware = new FunctionCallMiddleware( + functions: [functions.GetWeatherAsyncFunctionContract], + functionMap: new Dictionary>> + { + { functions.GetWeatherAsyncFunctionContract.Name!, functions.GetWeatherAsyncWrapper }, + }); + + // api-key is not required for local server + // so you can use any string here + var openAIClient = new OpenAIClient("api-key", option); + + var agent = new OpenAIChatAgent( + openAIClient: openAIClient, + name: "assistant", + modelName: "placeholder", + systemMessage: "You are a helpful AI assistant") + .RegisterMessageConnector() + .RegisterMiddleware(functionMiddleware) + .RegisterPrintMessage(); + + var reply = await agent.SendAsync("what's the weather in new york"); + #endregion Create_Agent + } +} From 5fd26091916d3902d30a26f9e54bc3cde2c4a022 Mon Sep 17 00:00:00 2001 From: William W Wang <107702013+wmwxwa@users.noreply.github.com> Date: Thu, 27 Jun 2024 19:34:10 -0400 Subject: [PATCH 11/57] Create azure_cosmos_db in ecosystems.md (#2371) * Create azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md * Update azure_cosmos_db.md --- website/docs/ecosystem/azure_cosmos_db.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 website/docs/ecosystem/azure_cosmos_db.md diff --git a/website/docs/ecosystem/azure_cosmos_db.md b/website/docs/ecosystem/azure_cosmos_db.md new file mode 100644 index 000000000000..1ec29e40d771 --- /dev/null +++ b/website/docs/ecosystem/azure_cosmos_db.md @@ -0,0 +1,13 @@ +# Azure Cosmos DB + +> "OpenAI relies on Cosmos DB to dynamically scale their ChatGPT service โ€“ one of the fastest-growing consumer apps ever โ€“ enabling high reliability and low maintenance." +> โ€“ Satya Nadella, Microsoft chairman and chief executive officer + +Azure Cosmos DB is a fully managed [NoSQL](https://learn.microsoft.com/en-us/azure/cosmos-db/distributed-nosql), [relational](https://learn.microsoft.com/en-us/azure/cosmos-db/distributed-relational), and [vector database](https://learn.microsoft.com/azure/cosmos-db/vector-database). It offers single-digit millisecond response times, automatic and instant scalability, along with guaranteed speed at any scale. Your business continuity is assured with up to 99.999% availability backed by SLA. + +Your can simplify your application development by using this single database service for all your AI agent memory system needs, from [geo-replicated distributed cache](https://medium.com/@marcodesanctis2/using-azure-cosmos-db-as-your-persistent-geo-replicated-distributed-cache-b381ad80f8a0) to tracing/logging to [vector database](https://learn.microsoft.com/en-us/azure/cosmos-db/vector-database). + +Learn more about how Azure Cosmos DB enhances the performance of your [AI agents](https://learn.microsoft.com/en-us/azure/cosmos-db/ai-agents). + +- [Try Azure Cosmos DB free](https://learn.microsoft.com/en-us/azure/cosmos-db/try-free) +- [Use Azure Cosmos DB lifetime free tier](https://learn.microsoft.com/en-us/azure/cosmos-db/free-tier) From 55cc542bcf62dd2e2b7afa1ddfc1a29df2740d5f Mon Sep 17 00:00:00 2001 From: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> Date: Fri, 28 Jun 2024 09:27:04 +0530 Subject: [PATCH 12/57] fix log_function_use warning (#3018) --- autogen/agentchat/conversable_agent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index b434fc648eb1..81c666de022c 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -2526,14 +2526,16 @@ def _wrap_function(self, func: F) -> F: @functools.wraps(func) def _wrapped_func(*args, **kwargs): retval = func(*args, **kwargs) - log_function_use(self, func, kwargs, retval) + if logging_enabled(): + log_function_use(self, func, kwargs, retval) return serialize_to_str(retval) @load_basemodels_if_needed @functools.wraps(func) async def _a_wrapped_func(*args, **kwargs): retval = await func(*args, **kwargs) - log_function_use(self, func, kwargs, retval) + if logging_enabled(): + log_function_use(self, func, kwargs, retval) return serialize_to_str(retval) wrapped_func = _a_wrapped_func if inspect.iscoroutinefunction(func) else _wrapped_func From 23c1dec206227f4eb30b00e15eebf32c0750e47e Mon Sep 17 00:00:00 2001 From: Mark Sze <66362098+marklysze@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:58:42 +1000 Subject: [PATCH 13/57] Groq Client (#3003) * Groq Client Class - main class and setup, except tests * Change pricing per K, added tests * Streaming support, including with tool calling * Used Groq retries instead of loop, thanks Gal-Gilor! * Fixed bug when using logging. --------- Co-authored-by: Qingyun Wu --- .github/workflows/contrib-tests.yml | 40 ++++ autogen/logger/file_logger.py | 3 +- autogen/logger/sqlite_logger.py | 3 +- autogen/oai/client.py | 15 +- autogen/oai/groq.py | 289 ++++++++++++++++++++++++++++ autogen/runtime_logging.py | 3 +- setup.py | 1 + test/oai/test_groq.py | 249 ++++++++++++++++++++++++ 8 files changed, 599 insertions(+), 4 deletions(-) create mode 100644 autogen/oai/groq.py create mode 100644 test/oai/test_groq.py diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index 7d8a932b0254..895e810022de 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -598,3 +598,43 @@ jobs: with: file: ./coverage.xml flags: unittests + + GroqTest: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ["3.9", "3.10", "3.11", "3.12"] + exclude: + - os: macos-latest + python-version: "3.9" + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for Groq + run: | + pip install -e .[groq,test] + - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash + run: | + if [[ ${{ matrix.os }} != ubuntu-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + - name: Coverage + run: | + pytest test/oai/test_groq.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.py index af5583587f66..cdebbdc0eb79 100644 --- a/autogen/logger/file_logger.py +++ b/autogen/logger/file_logger.py @@ -19,6 +19,7 @@ from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient from autogen.oai.gemini import GeminiClient + from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient from autogen.oai.together import TogetherClient @@ -204,7 +205,7 @@ def log_new_wrapper( def log_new_client( self, - client: AzureOpenAI | OpenAI | GeminiClient | AnthropicClient | MistralAIClient | TogetherClient, + client: AzureOpenAI | OpenAI | GeminiClient | AnthropicClient | MistralAIClient | TogetherClient | GroqClient, wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py index 969a943017e3..ccde6bd1d81b 100644 --- a/autogen/logger/sqlite_logger.py +++ b/autogen/logger/sqlite_logger.py @@ -20,6 +20,7 @@ from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient from autogen.oai.gemini import GeminiClient + from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient from autogen.oai.together import TogetherClient @@ -391,7 +392,7 @@ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[st def log_new_client( self, - client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient], + client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 2c14ca0d4a0c..a7b12ce83dad 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -70,6 +70,13 @@ except ImportError as e: together_import_exception = e +try: + from autogen.oai.groq import GroqClient + + groq_import_exception: Optional[ImportError] = None +except ImportError as e: + groq_import_exception = e + logger = logging.getLogger(__name__) if not logger.handlers: # Add the console handler. @@ -483,7 +490,13 @@ def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[s elif api_type is not None and api_type.startswith("together"): if together_import_exception: raise ImportError("Please install `together` to use the Together.AI API.") - self._clients.append(TogetherClient(**config)) + client = TogetherClient(**openai_config) + self._clients.append(client) + elif api_type is not None and api_type.startswith("groq"): + if groq_import_exception: + raise ImportError("Please install `groq` to use the Groq API.") + client = GroqClient(**openai_config) + self._clients.append(client) else: client = OpenAI(**openai_config) self._clients.append(OpenAIClient(client)) diff --git a/autogen/oai/groq.py b/autogen/oai/groq.py new file mode 100644 index 000000000000..a97240887c8e --- /dev/null +++ b/autogen/oai/groq.py @@ -0,0 +1,289 @@ +"""Create an OpenAI-compatible client using Groq's API. + +Example: + llm_config={ + "config_list": [{ + "api_type": "groq", + "model": "mixtral-8x7b-32768", + "api_key": os.environ.get("GROQ_API_KEY") + } + ]} + + agent = autogen.AssistantAgent("my_agent", llm_config=llm_config) + +Install Groq's python library using: pip install --upgrade groq + +Resources: +- https://console.groq.com/docs/quickstart +""" + +from __future__ import annotations + +import copy +import os +import time +import warnings +from typing import Any, Dict, List + +from groq import Groq, Stream +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.completion_usage import CompletionUsage + +from autogen.oai.client_utils import should_hide_tools, validate_parameter + +# Cost per thousand tokens - Input / Output (NOTE: Convert $/Million to $/K) +GROQ_PRICING_1K = { + "llama3-70b-8192": (0.00059, 0.00079), + "mixtral-8x7b-32768": (0.00024, 0.00024), + "llama3-8b-8192": (0.00005, 0.00008), + "gemma-7b-it": (0.00007, 0.00007), +} + + +class GroqClient: + """Client for Groq's API.""" + + def __init__(self, **kwargs): + """Requires api_key or environment variable to be set + + Args: + api_key (str): The API key for using Groq (or environment variable GROQ_API_KEY needs to be set) + """ + # Ensure we have the api_key upon instantiation + self.api_key = kwargs.get("api_key", None) + if not self.api_key: + self.api_key = os.getenv("GROQ_API_KEY") + + assert ( + self.api_key + ), "Please include the api_key in your config list entry for Groq or set the GROQ_API_KEY env variable." + + def message_retrieval(self, response) -> List: + """ + Retrieve and return a list of strings or a list of Choice.Message from the response. + + NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object, + since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used. + """ + return [choice.message for choice in response.choices] + + def cost(self, response) -> float: + return response.cost + + @staticmethod + def get_usage(response) -> Dict: + """Return usage summary of the response using RESPONSE_USAGE_KEYS.""" + # ... # pragma: no cover + return { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "cost": response.cost, + "model": response.model, + } + + def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Loads the parameters for Groq API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults""" + groq_params = {} + + # Check that we have what we need to use Groq's API + # We won't enforce the available models as they are likely to change + groq_params["model"] = params.get("model", None) + assert groq_params[ + "model" + ], "Please specify the 'model' in your config list entry to nominate the Groq model to use." + + # Validate allowed Groq parameters + # https://console.groq.com/docs/api-reference#chat + groq_params["frequency_penalty"] = validate_parameter( + params, "frequency_penalty", (int, float), True, None, (-2, 2), None + ) + groq_params["max_tokens"] = validate_parameter(params, "max_tokens", int, True, None, (0, None), None) + groq_params["presence_penalty"] = validate_parameter( + params, "presence_penalty", (int, float), True, None, (-2, 2), None + ) + groq_params["seed"] = validate_parameter(params, "seed", int, True, None, None, None) + groq_params["stream"] = validate_parameter(params, "stream", bool, True, False, None, None) + groq_params["temperature"] = validate_parameter(params, "temperature", (int, float), True, 1, (0, 2), None) + groq_params["top_p"] = validate_parameter(params, "top_p", (int, float), True, None, None, None) + + # Groq parameters not supported by their models yet, ignoring + # logit_bias, logprobs, top_logprobs + + # Groq parameters we are ignoring: + # n (must be 1), response_format (to enforce JSON but needs prompting as well), user, + # parallel_tool_calls (defaults to True), stop + # function_call (deprecated), functions (deprecated) + # tool_choice (none if no tools, auto if there are tools) + + return groq_params + + def create(self, params: Dict) -> ChatCompletion: + + messages = params.get("messages", []) + + # Convert AutoGen messages to Groq messages + groq_messages = oai_messages_to_groq_messages(messages) + + # Parse parameters to the Groq API's parameters + groq_params = self.parse_params(params) + + # Add tools to the call if we have them and aren't hiding them + if "tools" in params: + hide_tools = validate_parameter( + params, "hide_tools", str, False, "never", None, ["if_all_run", "if_any_run", "never"] + ) + if not should_hide_tools(groq_messages, params["tools"], hide_tools): + groq_params["tools"] = params["tools"] + + groq_params["messages"] = groq_messages + + # We use chat model by default, and set max_retries to 5 (in line with typical retries loop) + client = Groq(api_key=self.api_key, max_retries=5) + + # Token counts will be returned + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + # Streaming tool call recommendations + streaming_tool_calls = [] + + ans = None + try: + response = client.chat.completions.create(**groq_params) + except Exception as e: + raise RuntimeError(f"Groq exception occurred: {e}") + else: + + if groq_params["stream"]: + # Read in the chunks as they stream, taking in tool_calls which may be across + # multiple chunks if more than one suggested + ans = "" + for chunk in response: + ans = ans + (chunk.choices[0].delta.content or "") + + if chunk.choices[0].delta.tool_calls: + # We have a tool call recommendation + for tool_call in chunk.choices[0].delta.tool_calls: + streaming_tool_calls.append( + ChatCompletionMessageToolCall( + id=tool_call.id, + function={ + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + }, + type="function", + ) + ) + + if chunk.choices[0].finish_reason: + prompt_tokens = chunk.x_groq.usage.prompt_tokens + completion_tokens = chunk.x_groq.usage.completion_tokens + total_tokens = chunk.x_groq.usage.total_tokens + else: + # Non-streaming finished + ans: str = response.choices[0].message.content + + prompt_tokens = response.usage.prompt_tokens + completion_tokens = response.usage.completion_tokens + total_tokens = response.usage.total_tokens + + if response is not None: + + if isinstance(response, Stream): + # Streaming response + if chunk.choices[0].finish_reason == "tool_calls": + groq_finish = "tool_calls" + tool_calls = streaming_tool_calls + else: + groq_finish = "stop" + tool_calls = None + + response_content = ans + response_id = chunk.id + else: + # Non-streaming response + # If we have tool calls as the response, populate completed tool calls for our return OAI response + if response.choices[0].finish_reason == "tool_calls": + groq_finish = "tool_calls" + tool_calls = [] + for tool_call in response.choices[0].message.tool_calls: + tool_calls.append( + ChatCompletionMessageToolCall( + id=tool_call.id, + function={"name": tool_call.function.name, "arguments": tool_call.function.arguments}, + type="function", + ) + ) + else: + groq_finish = "stop" + tool_calls = None + + response_content = response.choices[0].message.content + response_id = response.id + else: + raise RuntimeError("Failed to get response from Groq after retrying 5 times.") + + # 3. convert output + message = ChatCompletionMessage( + role="assistant", + content=response_content, + function_call=None, + tool_calls=tool_calls, + ) + choices = [Choice(finish_reason=groq_finish, index=0, message=message)] + + response_oai = ChatCompletion( + id=response_id, + model=groq_params["model"], + created=int(time.time()), + object="chat.completion", + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + cost=calculate_groq_cost(prompt_tokens, completion_tokens, groq_params["model"]), + ) + + return response_oai + + +def oai_messages_to_groq_messages(messages: list[Dict[str, Any]]) -> list[dict[str, Any]]: + """Convert messages from OAI format to Groq's format. + We correct for any specific role orders and types. + """ + + groq_messages = copy.deepcopy(messages) + + # If we have a message with role='tool', which occurs when a function is executed, change it to 'user' + """ + for msg in together_messages: + if "role" in msg and msg["role"] == "tool": + msg["role"] = "user" + """ + + # Remove the name field + for message in groq_messages: + if "name" in message: + message.pop("name", None) + + return groq_messages + + +def calculate_groq_cost(input_tokens: int, output_tokens: int, model: str) -> float: + """Calculate the cost of the completion using the Groq pricing.""" + total = 0.0 + + if model in GROQ_PRICING_1K: + input_cost_per_k, output_cost_per_k = GROQ_PRICING_1K[model] + input_cost = (input_tokens / 1000) * input_cost_per_k + output_cost = (output_tokens / 1000) * output_cost_per_k + total = input_cost + output_cost + else: + warnings.warn(f"Cost calculation not available for model {model}", UserWarning) + + return total diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py index adb55ba63b4f..4ad76cf5b7da 100644 --- a/autogen/runtime_logging.py +++ b/autogen/runtime_logging.py @@ -15,6 +15,7 @@ from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient from autogen.oai.gemini import GeminiClient + from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient from autogen.oai.together import TogetherClient @@ -110,7 +111,7 @@ def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig def log_new_client( - client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient], + client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/setup.py b/setup.py index 738e09d9061c..9a67c70f49de 100644 --- a/setup.py +++ b/setup.py @@ -91,6 +91,7 @@ "long-context": ["llmlingua<0.3"], "anthropic": ["anthropic>=0.23.1"], "mistral": ["mistralai>=0.2.0"], + "groq": ["groq>=0.9.0"], } setuptools.setup( diff --git a/test/oai/test_groq.py b/test/oai/test_groq.py new file mode 100644 index 000000000000..f55edbd8c7a6 --- /dev/null +++ b/test/oai/test_groq.py @@ -0,0 +1,249 @@ +from unittest.mock import MagicMock, patch + +import pytest + +try: + from autogen.oai.groq import GroqClient, calculate_groq_cost + + skip = False +except ImportError: + GroqClient = object + InternalServerError = object + skip = True + + +# Fixtures for mock data +@pytest.fixture +def mock_response(): + class MockResponse: + def __init__(self, text, choices, usage, cost, model): + self.text = text + self.choices = choices + self.usage = usage + self.cost = cost + self.model = model + + return MockResponse + + +@pytest.fixture +def groq_client(): + return GroqClient(api_key="fake_api_key") + + +skip_reason = "Groq dependency is not installed" + + +# Test initialization and configuration +@pytest.mark.skipif(skip, reason=skip_reason) +def test_initialization(): + + # Missing any api_key + with pytest.raises(AssertionError) as assertinfo: + GroqClient() # Should raise an AssertionError due to missing api_key + + assert "Please include the api_key in your config list entry for Groq or set the GROQ_API_KEY env variable." in str( + assertinfo.value + ) + + # Creation works + GroqClient(api_key="fake_api_key") # Should create okay now. + + +# Test standard initialization +@pytest.mark.skipif(skip, reason=skip_reason) +def test_valid_initialization(groq_client): + assert groq_client.api_key == "fake_api_key", "Config api_key should be correctly set" + + +# Test parameters +@pytest.mark.skipif(skip, reason=skip_reason) +def test_parsing_params(groq_client): + # All parameters + params = { + "model": "llama3-8b-8192", + "frequency_penalty": 1.5, + "presence_penalty": 1.5, + "max_tokens": 1000, + "seed": 42, + "stream": False, + "temperature": 1, + "top_p": 0.8, + } + expected_params = { + "model": "llama3-8b-8192", + "frequency_penalty": 1.5, + "presence_penalty": 1.5, + "max_tokens": 1000, + "seed": 42, + "stream": False, + "temperature": 1, + "top_p": 0.8, + } + result = groq_client.parse_params(params) + assert result == expected_params + + # Only model, others set as defaults + params = { + "model": "llama3-8b-8192", + } + expected_params = { + "model": "llama3-8b-8192", + "frequency_penalty": None, + "presence_penalty": None, + "max_tokens": None, + "seed": None, + "stream": False, + "temperature": 1, + "top_p": None, + } + result = groq_client.parse_params(params) + assert result == expected_params + + # Incorrect types, defaults should be set, will show warnings but not trigger assertions + params = { + "model": "llama3-8b-8192", + "frequency_penalty": "1.5", + "presence_penalty": "1.5", + "max_tokens": "1000", + "seed": "42", + "stream": "False", + "temperature": "1", + "top_p": "0.8", + } + result = groq_client.parse_params(params) + assert result == expected_params + + # Values outside bounds, should warn and set to defaults + params = { + "model": "llama3-8b-8192", + "frequency_penalty": 5000, + "presence_penalty": -500, + "temperature": 3, + } + result = groq_client.parse_params(params) + assert result == expected_params + + # No model + params = { + "frequency_penalty": 1, + } + + with pytest.raises(AssertionError) as assertinfo: + result = groq_client.parse_params(params) + + assert "Please specify the 'model' in your config list entry to nominate the Groq model to use." in str( + assertinfo.value + ) + + +# Test cost calculation +@pytest.mark.skipif(skip, reason=skip_reason) +def test_cost_calculation(mock_response): + response = mock_response( + text="Example response", + choices=[{"message": "Test message 1"}], + usage={"prompt_tokens": 500, "completion_tokens": 300, "total_tokens": 800}, + cost=None, + model="llama3-70b-8192", + ) + assert ( + calculate_groq_cost(response.usage["prompt_tokens"], response.usage["completion_tokens"], response.model) + == 0.000532 + ), "Cost for this should be $0.000532" + + +# Test text generation +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.groq.GroqClient.create") +def test_create_response(mock_chat, groq_client): + # Mock GroqClient.chat response + mock_groq_response = MagicMock() + mock_groq_response.choices = [ + MagicMock(finish_reason="stop", message=MagicMock(content="Example Groq response", tool_calls=None)) + ] + mock_groq_response.id = "mock_groq_response_id" + mock_groq_response.model = "llama3-70b-8192" + mock_groq_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20) # Example token usage + + mock_chat.return_value = mock_groq_response + + # Test parameters + params = { + "messages": [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "World"}], + "model": "llama3-70b-8192", + } + + # Call the create method + response = groq_client.create(params) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.content == "Example Groq response" + ), "Response content should match expected output" + assert response.id == "mock_groq_response_id", "Response ID should match the mocked response ID" + assert response.model == "llama3-70b-8192", "Response model should match the mocked response model" + assert response.usage.prompt_tokens == 10, "Response prompt tokens should match the mocked response usage" + assert response.usage.completion_tokens == 20, "Response completion tokens should match the mocked response usage" + + +# Test functions/tools +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.groq.GroqClient.create") +def test_create_response_with_tool_call(mock_chat, groq_client): + # Mock `groq_response = client.chat(**groq_params)` + mock_function = MagicMock(name="currency_calculator") + mock_function.name = "currency_calculator" + mock_function.arguments = '{"base_currency": "EUR", "quote_currency": "USD", "base_amount": 123.45}' + + mock_function_2 = MagicMock(name="get_weather") + mock_function_2.name = "get_weather" + mock_function_2.arguments = '{"location": "Chicago"}' + + mock_chat.return_value = MagicMock( + choices=[ + MagicMock( + finish_reason="tool_calls", + message=MagicMock( + content="Sample text about the functions", + tool_calls=[ + MagicMock(id="gdRdrvnHh", function=mock_function), + MagicMock(id="abRdrvnHh", function=mock_function_2), + ], + ), + ) + ], + id="mock_groq_response_id", + model="llama3-70b-8192", + usage=MagicMock(prompt_tokens=10, completion_tokens=20), + ) + + # Construct parameters + converted_functions = [ + { + "type": "function", + "function": { + "description": "Currency exchange calculator.", + "name": "currency_calculator", + "parameters": { + "type": "object", + "properties": { + "base_amount": {"type": "number", "description": "Amount of currency in base_currency"}, + }, + "required": ["base_amount"], + }, + }, + } + ] + groq_messages = [ + {"role": "user", "content": "How much is 123.45 EUR in USD?"}, + {"role": "assistant", "content": "World"}, + ] + + # Call the create method + response = groq_client.create({"messages": groq_messages, "tools": converted_functions, "model": "llama3-70b-8192"}) + + # Assertions to check if the functions and content are included in the response + assert response.choices[0].message.content == "Sample text about the functions" + assert response.choices[0].message.tool_calls[0].function.name == "currency_calculator" + assert response.choices[0].message.tool_calls[1].function.name == "get_weather" From b8a47a8d4221bbec72d5bdf0e683b7a546a4b0b4 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Sun, 30 Jun 2024 11:43:22 -0700 Subject: [PATCH 14/57] [.Net] fix #3014 by adding local model function call in dotnet website (#3044) * add instruction in ollama-litellm function call example * add tutorial * fix tests --- .../Example13_OpenAIAgent_JsonMode.cs | 67 +------------ ...nAIChatAgent_ConnectToThirdPartyBackend.cs | 61 +----------- .../Tool_Call_With_Ollama_And_LiteLLM.cs | 29 ++++-- .../AutoGen.OpenAI.Sample/Use_Json_Mode.cs | 67 +++++++++++++ .../AutoGen.OpenAI.Tests.csproj | 1 + dotnet/test/AutoGen.Tests/BasicSampleTest.cs | 5 - .../Function-call-with-ollama-and-litellm.md | 93 +++++++++++++++++++ ...nAIChatAgent-connect-to-third-party-api.md | 11 +-- .../articles/OpenAIChatAgent-use-json-mode.md | 11 +-- dotnet/website/articles/toc.yml | 2 + .../tutorial/Create-agent-with-tools.md | 5 +- 11 files changed, 199 insertions(+), 153 deletions(-) create mode 100644 dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs create mode 100644 dotnet/website/articles/Function-call-with-ollama-and-litellm.md diff --git a/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs b/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs index dadad7f00b99..9e5b91ecc120 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs @@ -1,68 +1,3 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Example13_OpenAIAgent_JsonMode.cs - -using System.Text.Json; -using System.Text.Json.Serialization; -using AutoGen.Core; -using AutoGen.OpenAI; -using AutoGen.OpenAI.Extension; -using Azure.AI.OpenAI; -using FluentAssertions; - -namespace AutoGen.BasicSample; - -public class Example13_OpenAIAgent_JsonMode -{ - public static async Task RunAsync() - { - #region create_agent - var config = LLMConfiguration.GetAzureOpenAIGPT3_5_Turbo(deployName: "gpt-35-turbo"); // json mode only works with 0125 and later model. - var apiKey = config.ApiKey; - var endPoint = new Uri(config.Endpoint); - - var openAIClient = new OpenAIClient(endPoint, new Azure.AzureKeyCredential(apiKey)); - var openAIClientAgent = new OpenAIChatAgent( - openAIClient: openAIClient, - name: "assistant", - modelName: config.DeploymentName, - systemMessage: "You are a helpful assistant designed to output JSON.", - seed: 0, // explicitly set a seed to enable deterministic output - responseFormat: ChatCompletionsResponseFormat.JsonObject) // set response format to JSON object to enable JSON mode - .RegisterMessageConnector() - .RegisterPrintMessage(); - #endregion create_agent - - #region chat_with_agent - var reply = await openAIClientAgent.SendAsync("My name is John, I am 25 years old, and I live in Seattle."); - - var person = JsonSerializer.Deserialize(reply.GetContent()); - Console.WriteLine($"Name: {person.Name}"); - Console.WriteLine($"Age: {person.Age}"); - - if (!string.IsNullOrEmpty(person.Address)) - { - Console.WriteLine($"Address: {person.Address}"); - } - - Console.WriteLine("Done."); - #endregion chat_with_agent - - person.Name.Should().Be("John"); - person.Age.Should().Be(25); - person.Address.Should().BeNullOrEmpty(); - } -} - -#region person_class -public class Person -{ - [JsonPropertyName("name")] - public string Name { get; set; } - - [JsonPropertyName("age")] - public int Age { get; set; } - - [JsonPropertyName("address")] - public string Address { get; set; } -} -#endregion person_class +// this example has been moved to https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs \ No newline at end of file diff --git a/dotnet/sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs b/dotnet/sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs index eb8bcb179be1..f676e22a2d45 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs @@ -1,62 +1,3 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs -#region using_statement -using AutoGen.Core; -using AutoGen.OpenAI; -using AutoGen.OpenAI.Extension; -using Azure.AI.OpenAI; -using Azure.Core.Pipeline; -#endregion using_statement - -namespace AutoGen.BasicSample; - -#region CustomHttpClientHandler -public sealed class CustomHttpClientHandler : HttpClientHandler -{ - private string _modelServiceUrl; - - public CustomHttpClientHandler(string modelServiceUrl) - { - _modelServiceUrl = modelServiceUrl; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - request.RequestUri = new Uri($"{_modelServiceUrl}{request.RequestUri.PathAndQuery}"); - - return base.SendAsync(request, cancellationToken); - } -} -#endregion CustomHttpClientHandler - -public class Example16_OpenAIChatAgent_ConnectToThirdPartyBackend -{ - public static async Task RunAsync() - { - #region create_agent - using var client = new HttpClient(new CustomHttpClientHandler("http://localhost:11434")); - var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) - { - Transport = new HttpClientTransport(client), - }; - - // api-key is not required for local server - // so you can use any string here - var openAIClient = new OpenAIClient("api-key", option); - var model = "llama3"; - - var agent = new OpenAIChatAgent( - openAIClient: openAIClient, - name: "assistant", - modelName: model, - systemMessage: "You are a helpful assistant designed to output JSON.", - seed: 0) - .RegisterMessageConnector() - .RegisterPrintMessage(); - #endregion create_agent - - #region send_message - await agent.SendAsync("Can you write a piece of C# code to calculate 100th of fibonacci?"); - #endregion send_message - } -} +// this example has been moved to https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs \ No newline at end of file diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs index f4fabe3c9e83..b0b0adc0e6f5 100644 --- a/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs @@ -8,6 +8,7 @@ namespace AutoGen.OpenAI.Sample; +#region Function public partial class Function { [Function] @@ -16,18 +17,22 @@ public async Task GetWeatherAsync(string city) return await Task.FromResult("The weather in " + city + " is 72 degrees and sunny."); } } +#endregion Function + public class Tool_Call_With_Ollama_And_LiteLLM { public static async Task RunAsync() { - #region Create_Agent - var liteLLMUrl = "http://localhost:4000"; - using var httpClient = new HttpClient(new CustomHttpClientHandler(liteLLMUrl)); - var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) - { - Transport = new HttpClientTransport(httpClient), - }; + // Before running this code, make sure you have + // - Ollama: + // - Install dolphincoder:latest in Ollama + // - Ollama running on http://localhost:11434 + // - LiteLLM + // - Install LiteLLM + // - Start LiteLLM with the following command: + // - litellm --model ollama_chat/dolphincoder --port 4000 + # region Create_tools var functions = new Function(); var functionMiddleware = new FunctionCallMiddleware( functions: [functions.GetWeatherAsyncFunctionContract], @@ -35,6 +40,14 @@ public static async Task RunAsync() { { functions.GetWeatherAsyncFunctionContract.Name!, functions.GetWeatherAsyncWrapper }, }); + #endregion Create_tools + #region Create_Agent + var liteLLMUrl = "http://localhost:4000"; + using var httpClient = new HttpClient(new CustomHttpClientHandler(liteLLMUrl)); + var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) + { + Transport = new HttpClientTransport(httpClient), + }; // api-key is not required for local server // so you can use any string here @@ -43,7 +56,7 @@ public static async Task RunAsync() var agent = new OpenAIChatAgent( openAIClient: openAIClient, name: "assistant", - modelName: "placeholder", + modelName: "dolphincoder:latest", systemMessage: "You are a helpful AI assistant") .RegisterMessageConnector() .RegisterMiddleware(functionMiddleware) diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs new file mode 100644 index 000000000000..3bf88be72568 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs @@ -0,0 +1,67 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Example13_OpenAIAgent_JsonMode.cs + +using System.Text.Json; +using System.Text.Json.Serialization; +using AutoGen.Core; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using FluentAssertions; + +namespace AutoGen.BasicSample; + +public class Use_Json_Mode +{ + public static async Task RunAsync() + { + #region create_agent + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); + var model = "gpt-3.5-turbo"; + + var openAIClient = new OpenAIClient(apiKey); + var openAIClientAgent = new OpenAIChatAgent( + openAIClient: openAIClient, + name: "assistant", + modelName: model, + systemMessage: "You are a helpful assistant designed to output JSON.", + seed: 0, // explicitly set a seed to enable deterministic output + responseFormat: ChatCompletionsResponseFormat.JsonObject) // set response format to JSON object to enable JSON mode + .RegisterMessageConnector() + .RegisterPrintMessage(); + #endregion create_agent + + #region chat_with_agent + var reply = await openAIClientAgent.SendAsync("My name is John, I am 25 years old, and I live in Seattle."); + + var person = JsonSerializer.Deserialize(reply.GetContent()); + Console.WriteLine($"Name: {person.Name}"); + Console.WriteLine($"Age: {person.Age}"); + + if (!string.IsNullOrEmpty(person.Address)) + { + Console.WriteLine($"Address: {person.Address}"); + } + + Console.WriteLine("Done."); + #endregion chat_with_agent + + person.Name.Should().Be("John"); + person.Age.Should().Be(25); + person.Address.Should().BeNullOrEmpty(); + } +} + +#region person_class +public class Person +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("age")] + public int Age { get; set; } + + [JsonPropertyName("address")] + public string Address { get; set; } +} +#endregion person_class diff --git a/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj b/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj index ba499232beb9..04800a631ee6 100644 --- a/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj +++ b/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs index 8f2b9b2de51b..8da87d7902b9 100644 --- a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs +++ b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs @@ -37,11 +37,6 @@ public async Task AgentFunctionCallTestAsync() await Example03_Agent_FunctionCall.RunAsync(); } - [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] - public async Task OpenAIAgent_JsonMode() - { - await Example13_OpenAIAgent_JsonMode.RunAsync(); - } [ApiKeyFact("MISTRAL_API_KEY")] public async Task MistralClientAgent_TokenCount() diff --git a/dotnet/website/articles/Function-call-with-ollama-and-litellm.md b/dotnet/website/articles/Function-call-with-ollama-and-litellm.md new file mode 100644 index 000000000000..2dc595ba3adb --- /dev/null +++ b/dotnet/website/articles/Function-call-with-ollama-and-litellm.md @@ -0,0 +1,93 @@ +This example shows how to use function call with local LLM models where [Ollama](https://ollama.com/) as local model provider and [LiteLLM](https://docs.litellm.ai/docs/) proxy server which provides an openai-api compatible interface. + +[![](https://img.shields.io/badge/Open%20on%20Github-grey?logo=github)](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs) + +To run this example, the following prerequisites are required: +- Install [Ollama](https://ollama.com/) and [LiteLLM](https://docs.litellm.ai/docs/) on your local machine. +- A local model that supports function call. In this example `dolphincoder:latest` is used. + +## Install Ollama and pull `dolphincoder:latest` model +First, install Ollama by following the instructions on the [Ollama website](https://ollama.com/). + +After installing Ollama, pull the `dolphincoder:latest` model by running the following command: +```bash +ollama pull dolphincoder:latest +``` + +## Install LiteLLM and start the proxy server + +You can install LiteLLM by following the instructions on the [LiteLLM website](https://docs.litellm.ai/docs/). +```bash +pip install 'litellm[proxy]' +``` + +Then, start the proxy server by running the following command: + +```bash +litellm --model ollama_chat/dolphincoder --port 4000 +``` + +This will start an openai-api compatible proxy server at `http://localhost:4000`. You can verify if the server is running by observing the following output in the terminal: + +```bash +#------------------------------------------------------------# +# # +# 'The worst thing about this product is...' # +# https://github.com/BerriAI/litellm/issues/new # +# # +#------------------------------------------------------------# + +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit) +``` + +## Install AutoGen and AutoGen.SourceGenerator +In your project, install the AutoGen and AutoGen.SourceGenerator package using the following command: + +```bash +dotnet add package AutoGen +dotnet add package AutoGen.SourceGenerator +``` + +The `AutoGen.SourceGenerator` package is used to automatically generate type-safe `FunctionContract` instead of manually defining them. For more information, please check out [Create type-safe function](Create-type-safe-function-call.md). + +And in your project file, enable structural xml document support by setting the `GenerateDocumentationFile` property to `true`: + +```xml + + + true + +``` + +## Define `WeatherReport` function and create @AutoGen.Core.FunctionCallMiddleware + +Create a `public partial` class to host the methods you want to use in AutoGen agents. The method has to be a `public` instance method and its return type must be `Task`. After the methods are defined, mark them with `AutoGen.Core.FunctionAttribute` attribute. + +[!code-csharp[Define WeatherReport function](../../sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs?name=Function)] + +Then create a @AutoGen.Core.FunctionCallMiddleware and add the `WeatherReport` function to the middleware. The middleware will pass the `FunctionContract` to the agent when generating a response, and process the tool call response when receiving a `ToolCallMessage`. +[!code-csharp[Define WeatherReport function](../../sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs?name=Create_tools)] + +## Create @AutoGen.OpenAI.OpenAIChatAgent with `GetWeatherReport` tool and chat with it + +Because LiteLLM proxy server is openai-api compatible, we can use @AutoGen.OpenAI.OpenAIChatAgent to connect to it as a third-party openai-api provider. The agent is also registered with a @AutoGen.Core.FunctionCallMiddleware which contains the `WeatherReport` tool. Therefore, the agent can call the `WeatherReport` tool when generating a response. + +[!code-csharp[Create an agent with tools](../../sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs?name=Create_Agent)] + +The reply from the agent will similar to the following: +```bash +AggregateMessage from assistant +-------------------- +ToolCallMessage: +ToolCallMessage from assistant +-------------------- +- GetWeatherAsync: {"city": "new york"} +-------------------- + +ToolCallResultMessage: +ToolCallResultMessage from assistant +-------------------- +- GetWeatherAsync: The weather in new york is 72 degrees and sunny. +-------------------- +``` \ No newline at end of file diff --git a/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md b/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md index 8321fc87a5c2..0873765b1a6c 100644 --- a/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md +++ b/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md @@ -1,7 +1,6 @@ The following example shows how to connect to third-party OpenAI API using @AutoGen.OpenAI.OpenAIChatAgent. -> [!NOTE] -> You can find the complete code of this example in [Example16_OpenAIChatAgent_ConnectToThirdPartyBackend](https://github.com/microsoft/autogen/tree/main/dotnet/sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs). +[![](https://img.shields.io/badge/Open%20on%20Github-grey?logo=github)](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs) ## Overview A lot of LLM applications/platforms support spinning up a chat server that is compatible with OpenAI API, such as LM Studio, Ollama, Mistral etc. This means that you can connect to these servers using the @AutoGen.OpenAI.OpenAIChatAgent. @@ -25,24 +24,24 @@ ollama serve ## Steps - Import the required namespaces: -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=using_statement)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=using_statement)] - Create a `CustomHttpClientHandler` class. The `CustomHttpClientHandler` class is used to customize the HttpClientHandler. In this example, we override the `SendAsync` method to redirect the request to local Ollama server, which is running on `http://localhost:11434`. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=CustomHttpClientHandler)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=CustomHttpClientHandler)] - Create an `OpenAIChatAgent` instance and connect to the third-party API. Then create an @AutoGen.OpenAI.OpenAIChatAgent instance and connect to the OpenAI API from Ollama. You can customize the transport behavior of `OpenAIClient` by passing a customized `HttpClientTransport` instance. In the customized `HttpClientTransport` instance, we pass the `CustomHttpClientHandler` we just created which redirects all openai chat requests to the local Ollama server. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=create_agent)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=create_agent)] - Chat with the `OpenAIChatAgent`. Finally, you can start chatting with the agent. In this example, we send a coding question to the agent and get the response. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=send_message)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=send_message)] ## Sample Output The following is the sample output of the code snippet above: diff --git a/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md b/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md index a822cb046334..22f0ced00469 100644 --- a/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md +++ b/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md @@ -1,5 +1,7 @@ The following example shows how to enable JSON mode in @AutoGen.OpenAI.OpenAIChatAgent. +[![](https://img.shields.io/badge/Open%20on%20Github-grey?logo=github)](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs) + ## What is JSON mode? JSON mode is a new feature in OpenAI which allows you to instruct model to always respond with a valid JSON object. This is useful when you want to constrain the model output to JSON format only. @@ -8,20 +10,17 @@ JSON mode is a new feature in OpenAI which allows you to instruct model to alway ## How to enable JSON mode in OpenAIChatAgent. -> [!NOTE] -> You can find the complete example in the [Example13_OpenAIAgent_JsonMode](https://github.com/microsoft/autogen/tree/main/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs). - To enable JSON mode for @AutoGen.OpenAI.OpenAIChatAgent, set `responseFormat` to `ChatCompletionsResponseFormat.JsonObject` when creating the agent. Note that when enabling JSON mode, you also need to instruct the agent to output JSON format in its system message. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs?name=create_agent)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs?name=create_agent)] After enabling JSON mode, the `openAIClientAgent` will always respond in JSON format when it receives a message. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs?name=chat_with_agent)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs?name=chat_with_agent)] When running the example, the output from `openAIClientAgent` will be a valid JSON object which can be parsed as `Person` class defined below. Note that in the output, the `address` field is missing because the address information is not provided in user input. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs?name=person_class)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs?name=person_class)] The output will be: ```bash diff --git a/dotnet/website/articles/toc.yml b/dotnet/website/articles/toc.yml index 6d3787ae3d0b..2335ebf092b5 100644 --- a/dotnet/website/articles/toc.yml +++ b/dotnet/website/articles/toc.yml @@ -26,6 +26,8 @@ href: Create-type-safe-function-call.md - name: Use function call in an agent href: Use-function-call.md + - name: Function call with local model + href: Function-call-with-ollama-and-litellm.md - name: middleware items: - name: middleware overview diff --git a/dotnet/website/tutorial/Create-agent-with-tools.md b/dotnet/website/tutorial/Create-agent-with-tools.md index d3af7bff6f67..5d631890308a 100644 --- a/dotnet/website/tutorial/Create-agent-with-tools.md +++ b/dotnet/website/tutorial/Create-agent-with-tools.md @@ -98,7 +98,8 @@ Some LLM models support parallel tool call, which returns multiple tool calls in [!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=parallel_tool_call)] ## Further Reading -- [Function-call-with-openai](../articles/OpenAIChatAgent-use-function-call.md) -- [Function-call-with-gemini](../articles/AutoGen.Gemini/Function-call-with-gemini.md) +- [Function call with openai](../articles/OpenAIChatAgent-use-function-call.md) +- [Function call with gemini](../articles/AutoGen.Gemini/Function-call-with-gemini.md) +- [Function call with local model](../articles/Function-call-with-ollama-and-litellm.md) - [Use kernel plugin in other agents](../articles/AutoGen.SemanticKernel/Use-kernel-plugin-in-other-agents.md) - [function call in mistral](../articles/MistralChatAgent-use-function-call.md) \ No newline at end of file From e743d4d963834782d0e7b9dd28535633d4671ce9 Mon Sep 17 00:00:00 2001 From: Julia Kiseleva Date: Sun, 30 Jun 2024 15:00:13 -0700 Subject: [PATCH 15/57] Update README.md (#3025) adding links to blogposts to increase the clarity --- autogen/agentchat/contrib/agent_eval/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autogen/agentchat/contrib/agent_eval/README.md b/autogen/agentchat/contrib/agent_eval/README.md index 6588a1ec6113..478f28fd74ec 100644 --- a/autogen/agentchat/contrib/agent_eval/README.md +++ b/autogen/agentchat/contrib/agent_eval/README.md @@ -1,7 +1,9 @@ -Agents for running the AgentEval pipeline. +Agents for running the [AgentEval](https://microsoft.github.io/autogen/blog/2023/11/20/AgentEval/) pipeline. AgentEval is a process for evaluating a LLM-based system's performance on a given task. When given a task to evaluate and a few example runs, the critic and subcritic agents create evaluation criteria for evaluating a system's solution. Once the criteria has been created, the quantifier agent can evaluate subsequent task solutions based on the generated criteria. For more information see: [AgentEval Integration Roadmap](https://github.com/microsoft/autogen/issues/2162) + +See our [blog post](https://microsoft.github.io/autogen/blog/2024/06/21/AgentEval) for usage examples and general explanations. From 80ecbf900ca9c2bc7db519702ae132ec6b6c6a3f Mon Sep 17 00:00:00 2001 From: David Luong Date: Sun, 30 Jun 2024 19:21:34 -0400 Subject: [PATCH 16/57] [.Net] Support tools for AnthropicClient and AnthropicAgent (#2944) * Squash commits : support anthropic tools * Support tool_choice * Remove reference from TypeSafeFunctionCallCodeSnippet.cs and add own function in test proj --- .../AutoGen.Anthropic.Samples.csproj | 1 + ...icSamples.cs => Create_Anthropic_Agent.cs} | 2 +- .../Create_Anthropic_Agent_With_Tool.cs | 100 +++++++++++++++ .../AutoGen.Anthropic.Samples/Program.cs | 2 +- .../Agent/AnthropicClientAgent.cs | 11 +- .../src/AutoGen.Anthropic/AnthropicClient.cs | 107 ++++++++++++++-- .../Converters/ContentBaseConverter.cs | 4 + .../JsonPropertyNameEnumCoverter.cs | 44 +++++++ .../DTO/ChatCompletionRequest.cs | 8 ++ .../DTO/ChatCompletionResponse.cs | 6 +- dotnet/src/AutoGen.Anthropic/DTO/Content.cs | 28 +++++ dotnet/src/AutoGen.Anthropic/DTO/Tool.cs | 40 ++++++ .../src/AutoGen.Anthropic/DTO/ToolChoice.cs | 39 ++++++ .../Middleware/AnthropicMessageConnector.cs | 114 ++++++++++++++++-- .../AnthropicClientAgentTest.cs | 97 +++++++++++++++ .../AnthropicClientTest.cs | 52 ++++++++ .../AnthropicTestFunctionCalls.cs | 40 ++++++ .../AnthropicTestUtils.cs | 50 ++++++++ .../AutoGen.Anthropic.Tests.csproj | 1 + 19 files changed, 715 insertions(+), 31 deletions(-) rename dotnet/sample/AutoGen.Anthropic.Samples/{AnthropicSamples.cs => Create_Anthropic_Agent.cs} (95%) create mode 100644 dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs create mode 100644 dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs create mode 100644 dotnet/src/AutoGen.Anthropic/DTO/Tool.cs create mode 100644 dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs create mode 100644 dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj b/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj index 33a5aa7f16b6..2948c9bf283c 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj +++ b/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj @@ -13,6 +13,7 @@ + diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs similarity index 95% rename from dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs rename to dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs index 94b5f37511e6..031e50685488 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs @@ -7,7 +7,7 @@ namespace AutoGen.Anthropic.Samples; -public static class AnthropicSamples +public static class Create_Anthropic_Agent { public static async Task RunAsync() { diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs new file mode 100644 index 000000000000..26bd32dd12d5 --- /dev/null +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs @@ -0,0 +1,100 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Single_Anthropic_Tool.cs + +using AutoGen.Anthropic.DTO; +using AutoGen.Anthropic.Extensions; +using AutoGen.Anthropic.Utils; +using AutoGen.Core; +using FluentAssertions; + +namespace AutoGen.Anthropic.Samples; + +#region WeatherFunction + +public partial class WeatherFunction +{ + /// + /// Gets the weather based on the location and the unit + /// + /// + /// + /// + [Function] + public async Task GetWeather(string location, string unit) + { + // dummy implementation + return $"The weather in {location} is currently sunny with a tempature of {unit} (s)"; + } +} +#endregion +public class Create_Anthropic_Agent_With_Tool +{ + public static async Task RunAsync() + { + #region define_tool + var tool = new Tool + { + Name = "GetWeather", + Description = "Get the current weather in a given location", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { "location", new SchemaProperty { Type = "string", Description = "The city and state, e.g. San Francisco, CA" } }, + { "unit", new SchemaProperty { Type = "string", Description = "The unit of temperature, either \"celsius\" or \"fahrenheit\"" } } + }, + Required = new List { "location" } + } + }; + + var weatherFunction = new WeatherFunction(); + var functionMiddleware = new FunctionCallMiddleware( + functions: [ + weatherFunction.GetWeatherFunctionContract, + ], + functionMap: new Dictionary>> + { + { weatherFunction.GetWeatherFunctionContract.Name!, weatherFunction.GetWeatherWrapper }, + }); + + #endregion + + #region create_anthropic_agent + + var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? + throw new Exception("Missing ANTHROPIC_API_KEY environment variable."); + + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, apiKey); + var agent = new AnthropicClientAgent(anthropicClient, "assistant", AnthropicConstants.Claude3Haiku, + tools: [tool]); // Define tools for AnthropicClientAgent + #endregion + + #region register_middleware + + var agentWithConnector = agent + .RegisterMessageConnector() + .RegisterPrintMessage() + .RegisterStreamingMiddleware(functionMiddleware); + #endregion register_middleware + + #region single_turn + var question = new TextMessage(Role.Assistant, + "What is the weather like in San Francisco?", + from: "user"); + var functionCallReply = await agentWithConnector.SendAsync(question); + #endregion + + #region Single_turn_verify_reply + functionCallReply.Should().BeOfType(); + #endregion Single_turn_verify_reply + + #region Multi_turn + var finalReply = await agentWithConnector.SendAsync(chatHistory: [question, functionCallReply]); + #endregion Multi_turn + + #region Multi_turn_verify_reply + finalReply.Should().BeOfType(); + #endregion Multi_turn_verify_reply + } +} diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs index f3c615088610..6d1e4e594b99 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs @@ -7,6 +7,6 @@ internal static class Program { public static async Task Main(string[] args) { - await AnthropicSamples.RunAsync(); + await Create_Anthropic_Agent_With_Tool.RunAsync(); } } diff --git a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs index e395bb4a225f..173155a96a64 100644 --- a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs +++ b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs @@ -1,5 +1,6 @@ ๏ปฟusing System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -16,6 +17,8 @@ public class AnthropicClientAgent : IStreamingAgent private readonly string _systemMessage; private readonly decimal _temperature; private readonly int _maxTokens; + private readonly Tool[]? _tools; + private readonly ToolChoice? _toolChoice; public AnthropicClientAgent( AnthropicClient anthropicClient, @@ -23,7 +26,9 @@ public AnthropicClientAgent( string modelName, string systemMessage = "You are a helpful AI assistant", decimal temperature = 0.7m, - int maxTokens = 1024) + int maxTokens = 1024, + Tool[]? tools = null, + ToolChoice? toolChoice = null) { Name = name; _anthropicClient = anthropicClient; @@ -31,6 +36,8 @@ public AnthropicClientAgent( _systemMessage = systemMessage; _temperature = temperature; _maxTokens = maxTokens; + _tools = tools; + _toolChoice = toolChoice; } public async Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, @@ -59,6 +66,8 @@ private ChatCompletionRequest CreateParameters(IEnumerable messages, G Model = _modelName, Stream = shouldStream, Temperature = (decimal?)options?.Temperature ?? _temperature, + Tools = _tools?.ToList(), + ToolChoice = _toolChoice ?? ToolChoice.Auto }; chatCompletionRequest.Messages = BuildMessages(messages); diff --git a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs index 90bd33683f20..babcd5302aac 100644 --- a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs +++ b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs @@ -24,12 +24,12 @@ public sealed class AnthropicClient : IDisposable private static readonly JsonSerializerOptions JsonSerializerOptions = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Converters = { new ContentBaseConverter() } + Converters = { new ContentBaseConverter(), new JsonPropertyNameEnumConverter() } }; private static readonly JsonSerializerOptions JsonDeserializerOptions = new() { - Converters = { new ContentBaseConverter() } + Converters = { new ContentBaseConverter(), new JsonPropertyNameEnumConverter() } }; public AnthropicClient(HttpClient httpClient, string baseUrl, string apiKey) @@ -61,24 +61,64 @@ public async IAsyncEnumerable StreamingChatCompletionsAs using var reader = new StreamReader(await httpResponseMessage.Content.ReadAsStreamAsync()); var currentEvent = new SseEvent(); + while (await reader.ReadLineAsync() is { } line) { if (!string.IsNullOrEmpty(line)) { - currentEvent.Data = line.Substring("data:".Length).Trim(); + if (line.StartsWith("event:")) + { + currentEvent.EventType = line.Substring("event:".Length).Trim(); + } + else if (line.StartsWith("data:")) + { + currentEvent.Data = line.Substring("data:".Length).Trim(); + } } - else + else // an empty line indicates the end of an event { - if (currentEvent.Data == "[DONE]") - continue; + if (currentEvent.EventType == "content_block_start" && !string.IsNullOrEmpty(currentEvent.Data)) + { + var dataBlock = JsonSerializer.Deserialize(currentEvent.Data!); + if (dataBlock != null && dataBlock.ContentBlock?.Type == "tool_use") + { + currentEvent.ContentBlock = dataBlock.ContentBlock; + } + } - if (currentEvent.Data != null) + if (currentEvent.EventType is "message_start" or "content_block_delta" or "message_delta" && currentEvent.Data != null) { - yield return await JsonSerializer.DeserializeAsync( + var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), - cancellationToken: cancellationToken) ?? throw new Exception("Failed to deserialize response"); + cancellationToken: cancellationToken); + + if (res == null) + { + throw new Exception("Failed to deserialize response"); + } + + if (res.Delta?.Type == "input_json_delta" && !string.IsNullOrEmpty(res.Delta.PartialJson) && + currentEvent.ContentBlock != null) + { + currentEvent.ContentBlock.AppendDeltaParameters(res.Delta.PartialJson!); + } + else if (res.Delta is { StopReason: "tool_use" } && currentEvent.ContentBlock != null) + { + if (res.Content == null) + { + res.Content = [currentEvent.ContentBlock.CreateToolUseContent()]; + } + else + { + res.Content.Add(currentEvent.ContentBlock.CreateToolUseContent()); + } + + currentEvent = new SseEvent(); + } + + yield return res; } - else if (currentEvent.Data != null) + else if (currentEvent.EventType == "error" && currentEvent.Data != null) { var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), cancellationToken: cancellationToken); @@ -86,8 +126,10 @@ public async IAsyncEnumerable StreamingChatCompletionsAs throw new Exception(res?.Error?.Message); } - // Reset the current event for the next one - currentEvent = new SseEvent(); + if (currentEvent.ContentBlock == null) + { + currentEvent = new SseEvent(); + } } } } @@ -113,11 +155,50 @@ public void Dispose() private struct SseEvent { + public string EventType { get; set; } public string? Data { get; set; } + public ContentBlock? ContentBlock { get; set; } - public SseEvent(string? data = null) + public SseEvent(string eventType, string? data = null, ContentBlock? contentBlock = null) { + EventType = eventType; Data = data; + ContentBlock = contentBlock; } } + + private class ContentBlock + { + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("input")] + public object? Input { get; set; } + + public string? parameters { get; set; } + + public void AppendDeltaParameters(string deltaParams) + { + StringBuilder sb = new StringBuilder(parameters); + sb.Append(deltaParams); + parameters = sb.ToString(); + } + + public ToolUseContent CreateToolUseContent() + { + return new ToolUseContent { Id = Id, Name = Name, Input = parameters }; + } + } + + private class DataBlock + { + [JsonPropertyName("content_block")] + public ContentBlock? ContentBlock { get; set; } + } } diff --git a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs index 4cb8fdbb34e0..b41a761dc4d3 100644 --- a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs +++ b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs @@ -24,6 +24,10 @@ public override ContentBase Read(ref Utf8JsonReader reader, Type typeToConvert, return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); case "image": return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); + case "tool_use": + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); + case "tool_result": + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); } } diff --git a/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs new file mode 100644 index 000000000000..cd95d837cffd --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs @@ -0,0 +1,44 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// JsonPropertyNameEnumCoverter.cs + +using System; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AutoGen.Anthropic.Converters; + +internal class JsonPropertyNameEnumConverter : JsonConverter where T : struct, Enum +{ + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + string value = reader.GetString() ?? throw new JsonException("Value was null."); + + foreach (var field in typeToConvert.GetFields()) + { + var attribute = field.GetCustomAttribute(); + if (attribute?.Name == value) + { + return (T)Enum.Parse(typeToConvert, field.Name); + } + } + + throw new JsonException($"Unable to convert \"{value}\" to enum {typeToConvert}."); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + var field = value.GetType().GetField(value.ToString()); + var attribute = field.GetCustomAttribute(); + + if (attribute != null) + { + writer.WriteStringValue(attribute.Name); + } + else + { + writer.WriteStringValue(value.ToString()); + } + } +} + diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs index 0c1749eaa989..b18461e697bc 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs @@ -37,6 +37,12 @@ public class ChatCompletionRequest [JsonPropertyName("top_p")] public decimal? TopP { get; set; } + [JsonPropertyName("tools")] + public List? Tools { get; set; } + + [JsonPropertyName("tool_choice")] + public ToolChoice? ToolChoice { get; set; } + public ChatCompletionRequest() { Messages = new List(); @@ -62,4 +68,6 @@ public ChatMessage(string role, List content) Role = role; Content = content; } + + public void AddContent(ContentBase content) => Content.Add(content); } diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs index c6861f9c3150..2c6fa100fd63 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs @@ -49,9 +49,6 @@ public class StreamingMessage [JsonPropertyName("role")] public string? Role { get; set; } - [JsonPropertyName("content")] - public List? Content { get; set; } - [JsonPropertyName("model")] public string? Model { get; set; } @@ -85,6 +82,9 @@ public class Delta [JsonPropertyName("text")] public string? Text { get; set; } + [JsonPropertyName("partial_json")] + public string? PartialJson { get; set; } + [JsonPropertyName("usage")] public Usage? Usage { get; set; } } diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs index dd2481bd58f3..ee7a745a1416 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Content.cs +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace AutoGen.Anthropic.DTO; @@ -40,3 +41,30 @@ public class ImageSource [JsonPropertyName("data")] public string? Data { get; set; } } + +public class ToolUseContent : ContentBase +{ + [JsonPropertyName("type")] + public override string Type => "tool_use"; + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("input")] + public JsonNode? Input { get; set; } +} + +public class ToolResultContent : ContentBase +{ + [JsonPropertyName("type")] + public override string Type => "tool_result"; + + [JsonPropertyName("tool_use_id")] + public string? Id { get; set; } + + [JsonPropertyName("content")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs new file mode 100644 index 000000000000..41c20dc2a42d --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Tool.cs + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace AutoGen.Anthropic.DTO; + +public class Tool +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } + + [JsonPropertyName("input_schema")] + public InputSchema? InputSchema { get; set; } +} + +public class InputSchema +{ + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("properties")] + public Dictionary? Properties { get; set; } + + [JsonPropertyName("required")] + public List? Required { get; set; } +} + +public class SchemaProperty +{ + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } +} diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs b/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs new file mode 100644 index 000000000000..0a5c3790e1de --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs @@ -0,0 +1,39 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// ToolChoice.cs + +using System.Text.Json.Serialization; +using AutoGen.Anthropic.Converters; + +namespace AutoGen.Anthropic.DTO; + +[JsonConverter(typeof(JsonPropertyNameEnumConverter))] +public enum ToolChoiceType +{ + [JsonPropertyName("auto")] + Auto, // Default behavior + + [JsonPropertyName("any")] + Any, // Use any provided tool + + [JsonPropertyName("tool")] + Tool // Force a specific tool +} + +public class ToolChoice +{ + [JsonPropertyName("type")] + public ToolChoiceType Type { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + private ToolChoice(ToolChoiceType type, string? name = null) + { + Type = type; + Name = name; + } + + public static ToolChoice Auto => new(ToolChoiceType.Auto); + public static ToolChoice Any => new(ToolChoiceType.Any); + public static ToolChoice ToolUse(string name) => new(ToolChoiceType.Tool, name); +} diff --git a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs index bb2f5820f74c..f78ccd19deae 100644 --- a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs +++ b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using AutoGen.Anthropic.DTO; @@ -54,6 +55,17 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage chatMessage, IStreamingAgent agent) { + if (chatMessage.Content.Content is { Count: 1 } && + chatMessage.Content.Content[0] is ToolUseContent toolUseContent) + { + return new ToolCallMessage( + toolUseContent.Name ?? + throw new InvalidOperationException($"Expected {nameof(toolUseContent.Name)} to be specified"), + toolUseContent.Input?.ToString() ?? + throw new InvalidOperationException($"Expected {nameof(toolUseContent.Input)} to be specified"), + from: agent.Name); + } + var delta = chatMessage.Content.Delta; return delta != null && !string.IsNullOrEmpty(delta.Text) ? new TextMessageUpdate(role: Role.Assistant, delta.Text, from: agent.Name) @@ -71,16 +83,20 @@ private async Task> ProcessMessageAsync(IEnumerable ProcessTextMessage(textMessage, agent), ImageMessage imageMessage => - new MessageEnvelope(new ChatMessage("user", + (MessageEnvelope[])[new MessageEnvelope(new ChatMessage("user", new ContentBase[] { new ImageContent { Source = await ProcessImageSourceAsync(imageMessage) } } .ToList()), - from: agent.Name), + from: agent.Name)], MultiModalMessage multiModalMessage => await ProcessMultiModalMessageAsync(multiModalMessage, agent), - _ => message, + + ToolCallMessage toolCallMessage => ProcessToolCallMessage(toolCallMessage, agent), + ToolCallResultMessage toolCallResultMessage => ProcessToolCallResultMessage(toolCallResultMessage), + AggregateMessage toolCallAggregateMessage => ProcessToolCallAggregateMessage(toolCallAggregateMessage, agent), + _ => [message], }; - processedMessages.Add(processedMessage); + processedMessages.AddRange(processedMessage); } return processedMessages; @@ -93,15 +109,42 @@ private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from throw new ArgumentNullException(nameof(response.Content)); } - if (response.Content.Count != 1) + // When expecting a tool call, sometimes the response will contain two messages, one chat and one tool. + // The first message is typically a TextContent, of the LLM explaining what it is trying to do. + // The second message contains the tool call. + if (response.Content.Count > 1) { - throw new NotSupportedException($"{nameof(response.Content)} != 1"); + if (response.Content.Count == 2 && response.Content[0] is TextContent && + response.Content[1] is ToolUseContent toolUseContent) + { + return new ToolCallMessage(toolUseContent.Name ?? string.Empty, + toolUseContent.Input?.ToJsonString() ?? string.Empty, + from: from.Name); + } + + throw new NotSupportedException($"Expected {nameof(response.Content)} to have one output"); } - return new TextMessage(Role.Assistant, ((TextContent)response.Content[0]).Text ?? string.Empty, from: from.Name); + var content = response.Content[0]; + switch (content) + { + case TextContent textContent: + return new TextMessage(Role.Assistant, textContent.Text ?? string.Empty, from: from.Name); + + case ToolUseContent toolUseContent: + return new ToolCallMessage(toolUseContent.Name ?? string.Empty, + toolUseContent.Input?.ToJsonString() ?? string.Empty, + from: from.Name); + + case ImageContent: + throw new InvalidOperationException( + "Claude is an image understanding model only. It can interpret and analyze images, but it cannot generate, produce, edit, manipulate or create images"); + default: + throw new ArgumentOutOfRangeException(nameof(content)); + } } - private IMessage ProcessTextMessage(TextMessage textMessage, IAgent agent) + private IEnumerable> ProcessTextMessage(TextMessage textMessage, IAgent agent) { ChatMessage messages; @@ -139,10 +182,10 @@ private IMessage ProcessTextMessage(TextMessage textMessage, IAgent "user", textMessage.Content); } - return new MessageEnvelope(messages, from: textMessage.From); + return [new MessageEnvelope(messages, from: textMessage.From)]; } - private async Task ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent) + private async Task> ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent) { var content = new List(); foreach (var message in multiModalMessage.Content) @@ -158,8 +201,7 @@ private async Task ProcessMultiModalMessageAsync(MultiModalMessage mul } } - var chatMessage = new ChatMessage("user", content); - return MessageEnvelope.Create(chatMessage, agent.Name); + return [MessageEnvelope.Create(new ChatMessage("user", content), agent.Name)]; } private async Task ProcessImageSourceAsync(ImageMessage imageMessage) @@ -192,4 +234,52 @@ private async Task ProcessImageSourceAsync(ImageMessage imageMessag Data = Convert.ToBase64String(await response.Content.ReadAsByteArrayAsync()) }; } + + private IEnumerable ProcessToolCallMessage(ToolCallMessage toolCallMessage, IAgent agent) + { + var chatMessage = new ChatMessage("assistant", new List()); + foreach (var toolCall in toolCallMessage.ToolCalls) + { + chatMessage.AddContent(new ToolUseContent + { + Id = toolCall.ToolCallId, + Name = toolCall.FunctionName, + Input = JsonNode.Parse(toolCall.FunctionArguments) + }); + } + + return [MessageEnvelope.Create(chatMessage, toolCallMessage.From)]; + } + + private IEnumerable ProcessToolCallResultMessage(ToolCallResultMessage toolCallResultMessage) + { + var chatMessage = new ChatMessage("user", new List()); + foreach (var toolCall in toolCallResultMessage.ToolCalls) + { + chatMessage.AddContent(new ToolResultContent + { + Id = toolCall.ToolCallId ?? string.Empty, + Content = toolCall.Result, + }); + } + + return [MessageEnvelope.Create(chatMessage, toolCallResultMessage.From)]; + } + + private IEnumerable ProcessToolCallAggregateMessage(AggregateMessage aggregateMessage, IAgent agent) + { + if (aggregateMessage.From is { } from && from != agent.Name) + { + var contents = aggregateMessage.Message2.ToolCalls.Select(t => t.Result); + var messages = contents.Select(c => + new ChatMessage("assistant", c ?? throw new ArgumentNullException(nameof(c)))); + + return messages.Select(m => new MessageEnvelope(m, from: from)); + } + + var toolCallMessage = ProcessToolCallMessage(aggregateMessage.Message1, agent); + var toolCallResult = ProcessToolCallResultMessage(aggregateMessage.Message2); + + return toolCallMessage.Concat(toolCallResult); + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs index d29025b44aff..49cbb54af318 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs @@ -105,4 +105,101 @@ public async Task AnthropicAgentTestImageMessageAsync() reply.GetContent().Should().NotBeNullOrEmpty(); reply.From.Should().Be(agent.Name); } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentTestToolAsync() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var function = new TypeSafeFunctionCall(); + var functionCallMiddleware = new FunctionCallMiddleware( + functions: new[] { function.WeatherReportFunctionContract }, + functionMap: new Dictionary>> + { + { function.WeatherReportFunctionContract.Name ?? string.Empty, function.WeatherReportWrapper }, + }); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are an LLM that is specialized in finding the weather !", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector() + .RegisterStreamingMiddleware(functionCallMiddleware); + + var reply = await agent.SendAsync("What is the weather in Philadelphia?"); + reply.GetContent().Should().Be("Weather report for Philadelphia on today is sunny"); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentFunctionCallMessageTest() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant.", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector(); + + var weatherFunctionArgumets = """ + { + "city": "Philadelphia", + "date": "6/14/2024" + } + """; + + var function = new AnthropicTestFunctionCalls(); + var functionCallResult = await function.GetWeatherReportWrapper(weatherFunctionArgumets); + var toolCall = new ToolCall(function.WeatherReportFunctionContract.Name!, weatherFunctionArgumets) + { + ToolCallId = "get_weather", + Result = functionCallResult, + }; + + IMessage[] chatHistory = [ + new TextMessage(Role.User, "what's the weather in Philadelphia?"), + new ToolCallMessage([toolCall], from: "assistant"), + new ToolCallResultMessage([toolCall], from: "user" ), + ]; + + var reply = await agent.SendAsync(chatHistory: chatHistory); + + reply.Should().BeOfType(); + reply.GetContent().Should().Be("The weather report for Philadelphia on 6/14/2024 is sunny."); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentFunctionCallMiddlewareMessageTest() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + var function = new AnthropicTestFunctionCalls(); + var functionCallMiddleware = new FunctionCallMiddleware( + functions: [function.WeatherReportFunctionContract], + functionMap: new Dictionary>> + { + { function.WeatherReportFunctionContract.Name!, function.GetWeatherReportWrapper } + }); + + var functionCallAgent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant.", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector() + .RegisterStreamingMiddleware(functionCallMiddleware); + + var question = new TextMessage(Role.User, "what's the weather in Philadelphia?"); + var reply = await functionCallAgent.SendAsync(question); + + var finalReply = await functionCallAgent.SendAsync(chatHistory: [question, reply]); + finalReply.Should().BeOfType(); + finalReply.GetContent()!.ToLower().Should().Contain("sunny"); + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs index a0b1f60cfb95..66b7d007758a 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs @@ -1,5 +1,6 @@ ๏ปฟusing System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; using AutoGen.Anthropic.DTO; using AutoGen.Anthropic.Utils; @@ -108,6 +109,57 @@ public async Task AnthropicClientImageChatCompletionTestAsync() response.Usage.OutputTokens.Should().BeGreaterThan(0); } + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicClientTestToolsAsync() + { + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var request = new ChatCompletionRequest(); + request.Model = AnthropicConstants.Claude3Haiku; + request.Stream = false; + request.MaxTokens = 100; + request.Messages = new List() { new("user", "Use the stock price tool to look for MSFT. Your response should only be the tool.") }; + request.Tools = new List() { AnthropicTestUtils.StockTool }; + + ChatCompletionResponse response = + await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); + + Assert.NotNull(response.Content); + Assert.True(response.Content.First() is ToolUseContent); + ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); + Assert.Equal("get_stock_price", toolUseContent.Name); + Assert.NotNull(toolUseContent.Input); + Assert.True(toolUseContent.Input is JsonNode); + JsonNode jsonNode = toolUseContent.Input; + Assert.Equal("{\"ticker\":\"MSFT\"}", jsonNode.ToJsonString()); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicClientTestToolChoiceAsync() + { + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var request = new ChatCompletionRequest(); + request.Model = AnthropicConstants.Claude3Haiku; + request.Stream = false; + request.MaxTokens = 100; + request.Messages = new List() { new("user", "What is the weather today? Your response should only be the tool.") }; + request.Tools = new List() { AnthropicTestUtils.StockTool, AnthropicTestUtils.WeatherTool }; + + // Force to use get_stock_price even though the prompt is about weather + request.ToolChoice = ToolChoice.ToolUse("get_stock_price"); + + ChatCompletionResponse response = + await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); + + Assert.NotNull(response.Content); + Assert.True(response.Content.First() is ToolUseContent); + ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); + Assert.Equal("get_stock_price", toolUseContent.Name); + Assert.NotNull(toolUseContent.Input); + Assert.True(toolUseContent.Input is JsonNode); + } + private sealed class Person { [JsonPropertyName("name")] diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs new file mode 100644 index 000000000000..5f1c0971bf71 --- /dev/null +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs @@ -0,0 +1,40 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// AnthropicTestFunctions.cs + +using System.Text.Json; +using System.Text.Json.Serialization; +using AutoGen.Core; + +namespace AutoGen.Anthropic.Tests; + +public partial class AnthropicTestFunctionCalls +{ + private class GetWeatherSchema + { + [JsonPropertyName("city")] + public string? City { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + } + + /// + /// Get weather report + /// + /// city + /// date + [Function] + public async Task WeatherReport(string city, string date) + { + return $"Weather report for {city} on {date} is sunny"; + } + + public Task GetWeatherReportWrapper(string arguments) + { + var schema = JsonSerializer.Deserialize( + arguments, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + + return WeatherReport(schema?.City ?? string.Empty, schema?.Date ?? string.Empty); + } +} diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs index de630da6d87c..a1faffec5344 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs @@ -1,6 +1,8 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // AnthropicTestUtils.cs +using AutoGen.Anthropic.DTO; + namespace AutoGen.Anthropic.Tests; public static class AnthropicTestUtils @@ -13,4 +15,52 @@ public static async Task Base64FromImageAsync(string imageName) return Convert.ToBase64String( await File.ReadAllBytesAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "images", imageName))); } + + public static Tool WeatherTool + { + get + { + return new Tool + { + Name = "WeatherReport", + Description = "Get the current weather", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { "city", new SchemaProperty {Type = "string", Description = "The name of the city"} }, + { "date", new SchemaProperty {Type = "string", Description = "date of the day"} } + } + } + }; + } + } + + public static Tool StockTool + { + get + { + return new Tool + { + Name = "get_stock_price", + Description = "Get the current stock price for a given ticker symbol.", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { + "ticker", new SchemaProperty + { + Type = "string", + Description = "The stock ticker symbol, e.g. AAPL for Apple Inc." + } + } + }, + Required = new List { "ticker" } + } + }; + } + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj index 0f22d9fe6764..ac479ed2e722 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj +++ b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj @@ -12,6 +12,7 @@ + From 66e50618bda21da04da6245d5408a41664d346d0 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 1 Jul 2024 09:52:57 -0700 Subject: [PATCH 17/57] [.Net] Fix #3045 (#3047) * make IStreamingMessage obsolete * update final reply message --- .../Agent/AnthropicClientAgent.cs | 2 +- .../Middleware/AnthropicMessageConnector.cs | 6 +++--- dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs | 2 +- .../Agent/MiddlewareStreamingAgent.cs | 4 ++-- dotnet/src/AutoGen.Core/Message/IMessage.cs | 14 +++++++++----- .../src/AutoGen.Core/Message/MessageEnvelope.cs | 4 ++-- dotnet/src/AutoGen.Core/Message/TextMessage.cs | 4 ++-- .../src/AutoGen.Core/Message/ToolCallMessage.cs | 2 +- .../Middleware/FunctionCallMiddleware.cs | 16 ++++++++++------ .../Middleware/IStreamingMiddleware.cs | 2 +- .../Middleware/PrintMessageMiddleware.cs | 2 +- dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs | 2 +- .../Middleware/GeminiMessageConnector.cs | 2 +- .../AutoGen.Mistral/Agent/MistralClientAgent.cs | 2 +- .../Middleware/MistralChatMessageConnector.cs | 6 +++--- dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs | 2 +- .../Middlewares/OllamaMessageConnector.cs | 4 ++-- dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs | 2 +- .../src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs | 2 +- .../OpenAIChatRequestMessageConnector.cs | 6 +++--- .../SemanticKernelChatMessageContentConnector.cs | 8 ++++---- .../SemanticKernelAgent.cs | 2 +- .../AutoGen.Gemini.Tests/GeminiAgentTests.cs | 8 ++++---- .../AutoGen.Gemini.Tests/GeminiMessageTests.cs | 4 ++-- .../VertexGeminiClientTests.cs | 2 +- .../AutoGen.Ollama.Tests/OllamaAgentTests.cs | 8 ++++---- .../AutoGen.Ollama.Tests/OllamaMessageTests.cs | 4 ++-- dotnet/test/AutoGen.Tests/BasicSampleTest.cs | 6 ------ dotnet/test/AutoGen.Tests/EchoAgent.cs | 2 +- dotnet/test/AutoGen.Tests/SingleAgentTest.cs | 2 +- 30 files changed, 67 insertions(+), 65 deletions(-) diff --git a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs index 173155a96a64..bf05ee974445 100644 --- a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs +++ b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs @@ -47,7 +47,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return new MessageEnvelope(response, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (var message in _anthropicClient.StreamingChatCompletionsAsync( diff --git a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs index f78ccd19deae..af06a0547849 100644 --- a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs +++ b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs @@ -29,7 +29,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, : response; } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = context.Messages; @@ -37,7 +37,7 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)) { - if (reply is IStreamingMessage chatMessage) + if (reply is IMessage chatMessage) { var response = ProcessChatCompletionResponse(chatMessage, agent); if (response is not null) @@ -52,7 +52,7 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c } } - private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage chatMessage, + private IMessage? ProcessChatCompletionResponse(IMessage chatMessage, IStreamingAgent agent) { if (chatMessage.Content.Content is { Count: 1 } && diff --git a/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs index 665f18bac12a..6b7794c921ad 100644 --- a/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs @@ -11,7 +11,7 @@ namespace AutoGen.Core; /// public interface IStreamingAgent : IAgent { - public IAsyncEnumerable GenerateStreamingReplyAsync( + public IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs index 52967d6ff1ce..c7643b1e4735 100644 --- a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs @@ -47,7 +47,7 @@ public Task GenerateReplyAsync(IEnumerable messages, Generat return _agent.GenerateReplyAsync(messages, options, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) { return _agent.GenerateStreamingReplyAsync(messages, options, cancellationToken); } @@ -83,7 +83,7 @@ public Task GenerateReplyAsync(IEnumerable messages, Generat return this.streamingMiddleware.InvokeAsync(context, (IAgent)innerAgent, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) { if (streamingMiddleware is null) { diff --git a/dotnet/src/AutoGen.Core/Message/IMessage.cs b/dotnet/src/AutoGen.Core/Message/IMessage.cs index ad215d510e3b..9952cbf06792 100644 --- a/dotnet/src/AutoGen.Core/Message/IMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/IMessage.cs @@ -1,6 +1,7 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // IMessage.cs +using System; using System.Collections.Generic; namespace AutoGen.Core; @@ -35,19 +36,21 @@ namespace AutoGen.Core; /// /// /// -public interface IMessage : IStreamingMessage +public interface IMessage { + string? From { get; set; } } -public interface IMessage : IMessage, IStreamingMessage +public interface IMessage : IMessage { + T Content { get; } } /// /// The interface for messages that can get text content. /// This interface will be used by to get the content from the message. /// -public interface ICanGetTextContent : IMessage, IStreamingMessage +public interface ICanGetTextContent : IMessage { public string? GetContent(); } @@ -55,17 +58,18 @@ public interface ICanGetTextContent : IMessage, IStreamingMessage /// /// The interface for messages that can get a list of /// -public interface ICanGetToolCalls : IMessage, IStreamingMessage +public interface ICanGetToolCalls : IMessage { public IEnumerable GetToolCalls(); } - +[Obsolete("Use IMessage instead")] public interface IStreamingMessage { string? From { get; set; } } +[Obsolete("Use IMessage instead")] public interface IStreamingMessage : IStreamingMessage { T Content { get; } diff --git a/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs b/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs index f83bea279260..dc9709bbde5b 100644 --- a/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs +++ b/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs @@ -5,7 +5,7 @@ namespace AutoGen.Core; -public abstract class MessageEnvelope : IMessage, IStreamingMessage +public abstract class MessageEnvelope : IMessage { public MessageEnvelope(string? from = null, IDictionary? metadata = null) { @@ -23,7 +23,7 @@ public static MessageEnvelope Create(TContent content, strin public IDictionary Metadata { get; set; } } -public class MessageEnvelope : MessageEnvelope, IMessage, IStreamingMessage +public class MessageEnvelope : MessageEnvelope, IMessage { public MessageEnvelope(T content, string? from = null, IDictionary? metadata = null) : base(from, metadata) diff --git a/dotnet/src/AutoGen.Core/Message/TextMessage.cs b/dotnet/src/AutoGen.Core/Message/TextMessage.cs index addd8728a926..9419c2b3ba86 100644 --- a/dotnet/src/AutoGen.Core/Message/TextMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/TextMessage.cs @@ -3,7 +3,7 @@ namespace AutoGen.Core; -public class TextMessage : IMessage, IStreamingMessage, ICanGetTextContent +public class TextMessage : IMessage, ICanGetTextContent { public TextMessage(Role role, string content, string? from = null) { @@ -51,7 +51,7 @@ public override string ToString() } } -public class TextMessageUpdate : IStreamingMessage, ICanGetTextContent +public class TextMessageUpdate : IMessage, ICanGetTextContent { public TextMessageUpdate(Role role, string? content, string? from = null) { diff --git a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs index d0f89e1ecdde..8660b323044f 100644 --- a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs @@ -109,7 +109,7 @@ public IEnumerable GetToolCalls() } } -public class ToolCallMessageUpdate : IStreamingMessage +public class ToolCallMessageUpdate : IMessage { public ToolCallMessageUpdate(string functionName, string functionArgumentUpdate, string? from = null) { diff --git a/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs index d0788077b590..7d30f6d0928a 100644 --- a/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs @@ -70,7 +70,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return reply; } - public async IAsyncEnumerable InvokeAsync( + public async IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -86,16 +86,16 @@ public async IAsyncEnumerable InvokeAsync( var combinedFunctions = this.functions?.Concat(options.Functions ?? []) ?? options.Functions; options.Functions = combinedFunctions?.ToArray(); - IStreamingMessage? initMessage = default; + IMessage? mergedFunctionCallMessage = default; await foreach (var message in agent.GenerateStreamingReplyAsync(context.Messages, options, cancellationToken)) { if (message is ToolCallMessageUpdate toolCallMessageUpdate && this.functionMap != null) { - if (initMessage is null) + if (mergedFunctionCallMessage is null) { - initMessage = new ToolCallMessage(toolCallMessageUpdate); + mergedFunctionCallMessage = new ToolCallMessage(toolCallMessageUpdate); } - else if (initMessage is ToolCallMessage toolCall) + else if (mergedFunctionCallMessage is ToolCallMessage toolCall) { toolCall.Update(toolCallMessageUpdate); } @@ -104,13 +104,17 @@ public async IAsyncEnumerable InvokeAsync( throw new InvalidOperationException("The first message is ToolCallMessage, but the update message is not ToolCallMessageUpdate"); } } + else if (message is ToolCallMessage toolCallMessage1) + { + mergedFunctionCallMessage = toolCallMessage1; + } else { yield return message; } } - if (initMessage is ToolCallMessage toolCallMsg) + if (mergedFunctionCallMessage is ToolCallMessage toolCallMsg) { yield return await this.InvokeToolCallMessagesAfterInvokingAgentAsync(toolCallMsg, agent); } diff --git a/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs index bc7aec57f52b..d550bdb519ce 100644 --- a/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs @@ -14,7 +14,7 @@ public interface IStreamingMiddleware : IMiddleware /// /// The streaming version of . /// - public IAsyncEnumerable InvokeAsync( + public IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, CancellationToken cancellationToken = default); diff --git a/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs index 099f78e5f176..a4e84de85a44 100644 --- a/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs @@ -48,7 +48,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, } } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { IMessage? recentUpdate = null; await foreach (var message in agent.GenerateStreamingReplyAsync(context.Messages, context.Options, cancellationToken)) diff --git a/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs b/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs index b081faae8321..e759ba26d1e9 100644 --- a/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs +++ b/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs @@ -143,7 +143,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return MessageEnvelope.Create(response, this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var request = BuildChatRequest(messages, options); var response = this.client.GenerateContentStreamAsync(request); diff --git a/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs b/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs index cb18ba084d78..422fb4cd3458 100644 --- a/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs +++ b/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs @@ -39,7 +39,7 @@ public GeminiMessageConnector(bool strictMode = false) public string Name => nameof(GeminiMessageConnector); - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = ProcessMessage(context.Messages, agent); diff --git a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs index cc2c74145504..ac144854fac7 100644 --- a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs +++ b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs @@ -78,7 +78,7 @@ public async Task GenerateReplyAsync( return new MessageEnvelope(response, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs b/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs index 95592e97fcc5..78de12a5c01e 100644 --- a/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs +++ b/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs @@ -15,14 +15,14 @@ public class MistralChatMessageConnector : IStreamingMiddleware, IMiddleware { public string? Name => nameof(MistralChatMessageConnector); - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = context.Messages; var chatMessages = ProcessMessage(messages, agent); var chunks = new List(); await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)) { - if (reply is IStreamingMessage chatMessage) + if (reply is IMessage chatMessage) { chunks.Add(chatMessage.Content); var response = ProcessChatCompletionResponse(chatMessage, agent); @@ -167,7 +167,7 @@ private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from } } - private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage message, IAgent agent) + private IMessage? ProcessChatCompletionResponse(IMessage message, IAgent agent) { var response = message.Content; if (response.VarObject != "chat.completion.chunk") diff --git a/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs b/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs index 9ef68388d605..87b176d8bcc5 100644 --- a/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs +++ b/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs @@ -53,7 +53,7 @@ public async Task GenerateReplyAsync( } } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs b/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs index a21ec3a1c991..3919b238d659 100644 --- a/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs +++ b/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs @@ -30,14 +30,14 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, }; } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = ProcessMessage(context.Messages, agent); var chunks = new List(); await foreach (var update in agent.GenerateStreamingReplyAsync(messages, context.Options, cancellationToken)) { - if (update is IStreamingMessage chatResponseUpdate) + if (update is IMessage chatResponseUpdate) { var response = chatResponseUpdate.Content switch { diff --git a/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs b/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs index cdc6cc464d17..5de481245b72 100644 --- a/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs +++ b/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs @@ -104,7 +104,7 @@ public async Task GenerateReplyAsync( return await _innerAgent.GenerateReplyAsync(messages, options, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync( + public IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs b/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs index 37a4882f69e1..b192cde1024b 100644 --- a/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs +++ b/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs @@ -87,7 +87,7 @@ public async Task GenerateReplyAsync( return new MessageEnvelope(reply, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs index c1dc2caa99fb..e1dd0757fcf3 100644 --- a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs +++ b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs @@ -47,7 +47,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return PostProcessMessage(reply); } - public async IAsyncEnumerable InvokeAsync( + public async IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -57,7 +57,7 @@ public async IAsyncEnumerable InvokeAsync( string? currentToolName = null; await foreach (var reply in streamingReply) { - if (reply is IStreamingMessage update) + if (reply is IMessage update) { if (update.Content.FunctionName is string functionName) { @@ -98,7 +98,7 @@ public IMessage PostProcessMessage(IMessage message) }; } - public IStreamingMessage? PostProcessStreamingMessage(IStreamingMessage update, string? currentToolName) + public IMessage? PostProcessStreamingMessage(IMessage update, string? currentToolName) { if (update.Content.ContentUpdate is string contentUpdate) { diff --git a/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs b/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs index 6ce242eb1abe..a055c0afcb6a 100644 --- a/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs +++ b/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs @@ -47,7 +47,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return PostProcessMessage(reply); } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var chatMessageContents = ProcessMessage(context.Messages, agent) .Select(m => new MessageEnvelope(m)); @@ -67,11 +67,11 @@ private IMessage PostProcessMessage(IMessage input) }; } - private IStreamingMessage PostProcessStreamingMessage(IStreamingMessage input) + private IMessage PostProcessStreamingMessage(IMessage input) { return input switch { - IStreamingMessage streamingMessage => PostProcessMessage(streamingMessage), + IMessage streamingMessage => PostProcessMessage(streamingMessage), IMessage msg => PostProcessMessage(msg), _ => input, }; @@ -98,7 +98,7 @@ private IMessage PostProcessMessage(IMessage messageEnvelope } } - private IStreamingMessage PostProcessMessage(IStreamingMessage streamingMessage) + private IMessage PostProcessMessage(IMessage streamingMessage) { var chatMessageContent = streamingMessage.Content; if (chatMessageContent.ChoiceIndex > 0) diff --git a/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs b/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs index 21f652f56c4f..d12c54c1b3b2 100644 --- a/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs +++ b/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs @@ -65,7 +65,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return new MessageEnvelope(reply.First(), from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs index 872cce5e645b..6d095845f803 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs @@ -86,8 +86,8 @@ public async Task VertexGeminiAgentGenerateStreamingReplyForTextContentAsync() var message = MessageEnvelope.Create(textContent, from: agent.Name); var completion = agent.GenerateStreamingReplyAsync([message]); - var chunks = new List(); - IStreamingMessage finalReply = null!; + var chunks = new List(); + IMessage finalReply = null!; await foreach (var item in completion) { @@ -212,8 +212,8 @@ public async Task VertexGeminiAgentGenerateStreamingReplyWithToolsAsync() var message = MessageEnvelope.Create(textContent, from: agent.Name); - var chunks = new List(); - IStreamingMessage finalReply = null!; + var chunks = new List(); + IMessage finalReply = null!; var completion = agent.GenerateStreamingReplyAsync([message]); diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs index 7ffb532ea9c1..12ba94734032 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs @@ -225,10 +225,10 @@ public async Task ItProcessStreamingTextMessageAsync() }) .Select(m => MessageEnvelope.Create(m)); - IStreamingMessage? finalReply = null; + IMessage? finalReply = null; await foreach (var reply in agent.GenerateStreamingReplyAsync(messageChunks)) { - reply.Should().BeAssignableTo(); + reply.Should().BeAssignableTo(); finalReply = reply; } diff --git a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs index 2f06305ed59f..8063b7077035 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs @@ -53,7 +53,7 @@ public async Task ItGenerateContentWithImageAsync() var model = "gemini-1.5-flash-001"; var text = "what's in the image"; - var imagePath = Path.Combine("testData", "images", "image.png"); + var imagePath = Path.Combine("testData", "images", "square.png"); var image = File.ReadAllBytes(imagePath); var request = new GenerateContentRequest { diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs index c1fb466f0b09..8a416116ea92 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs @@ -65,8 +65,8 @@ public async Task GenerateStreamingReplyAsync_ReturnsValidMessages_WhenCalled() var msg = new Message("user", "hey how are you"); var messages = new IMessage[] { MessageEnvelope.Create(msg, from: modelName) }; - IStreamingMessage? finalReply = default; - await foreach (IStreamingMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) + IMessage? finalReply = default; + await foreach (IMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) { message.Should().NotBeNull(); message.From.Should().Be(ollamaAgent.Name); @@ -171,8 +171,8 @@ public async Task ItReturnValidStreamingMessageUsingLLavaAsync() var messages = new IMessage[] { MessageEnvelope.Create(imageMessage, from: modelName) }; - IStreamingMessage? finalReply = default; - await foreach (IStreamingMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) + IMessage? finalReply = default; + await foreach (IMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) { message.Should().NotBeNull(); message.From.Should().Be(ollamaAgent.Name); diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs index b19291e97671..82cc462061da 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs @@ -57,10 +57,10 @@ public async Task ItProcessStreamingTextMessageAsync() }) .Select(m => MessageEnvelope.Create(m)); - IStreamingMessage? finalReply = null; + IMessage? finalReply = null; await foreach (var reply in agent.GenerateStreamingReplyAsync(messageChunks)) { - reply.Should().BeAssignableTo(); + reply.Should().BeAssignableTo(); finalReply = reply; } diff --git a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs index 8da87d7902b9..89925b7d3b39 100644 --- a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs +++ b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs @@ -44,12 +44,6 @@ public async Task MistralClientAgent_TokenCount() await Example14_MistralClientAgent_TokenCount.RunAsync(); } - [ApiKeyFact("OPENAI_API_KEY")] - public async Task DynamicGroupChatGetMLNetPRTestAsync() - { - await Example04_Dynamic_GroupChat_Coding_Task.RunAsync(); - } - [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] public async Task DynamicGroupChatCalculateFibonacciAsync() { diff --git a/dotnet/test/AutoGen.Tests/EchoAgent.cs b/dotnet/test/AutoGen.Tests/EchoAgent.cs index 9cead5ad2516..af5490218e8d 100644 --- a/dotnet/test/AutoGen.Tests/EchoAgent.cs +++ b/dotnet/test/AutoGen.Tests/EchoAgent.cs @@ -29,7 +29,7 @@ public Task GenerateReplyAsync( return Task.FromResult(lastMessage); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { foreach (var message in messages) { diff --git a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs index 5a3a9734cd1a..64bdc062eb73 100644 --- a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs +++ b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs @@ -297,7 +297,7 @@ public async Task EchoFunctionCallExecutionStreamingTestAsync(IStreamingAgent ag }; var replyStream = agent.GenerateStreamingReplyAsync(messages: new[] { helloWorld }, option); var answer = "[ECHO] Hello world"; - IStreamingMessage? finalReply = default; + IMessage? finalReply = default; await foreach (var reply in replyStream) { reply.From.Should().Be(agent.Name); From 8c56cc7d8092711514efc5bb32964bb1844414ba Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 2 Jul 2024 21:55:32 +0800 Subject: [PATCH 18/57] Fix llama_index tests (#3063) --- autogen/agentchat/contrib/llamaindex_conversable_agent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/autogen/agentchat/contrib/llamaindex_conversable_agent.py b/autogen/agentchat/contrib/llamaindex_conversable_agent.py index f7a9c3e615dc..dbf6f274ae87 100644 --- a/autogen/agentchat/contrib/llamaindex_conversable_agent.py +++ b/autogen/agentchat/contrib/llamaindex_conversable_agent.py @@ -8,15 +8,14 @@ try: from llama_index.core.agent.runner.base import AgentRunner + from llama_index.core.base.llms.types import ChatMessage from llama_index.core.chat_engine.types import AgentChatResponse - from llama_index_client import ChatMessage except ImportError as e: logger.fatal("Failed to import llama-index. Try running 'pip install llama-index'") raise e class LLamaIndexConversableAgent(ConversableAgent): - def __init__( self, name: str, From 62792478a4927b88585bcb187f73695fe26b0d7a Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Wed, 3 Jul 2024 00:00:31 +0800 Subject: [PATCH 19/57] Update qdrant dependency (#3064) * Update qdrant dependency * Update qdrant dependency --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 9a67c70f49de..cfaad885968c 100644 --- a/setup.py +++ b/setup.py @@ -72,10 +72,7 @@ "mathchat": ["sympy", "pydantic==1.10.9", "wolframalpha"], "retrievechat": retrieve_chat, "retrievechat-pgvector": retrieve_chat_pgvector, - "retrievechat-qdrant": [ - *retrieve_chat, - "qdrant_client[fastembed]<1.9.2", - ], + "retrievechat-qdrant": [*retrieve_chat, "qdrant_client", "fastembed>=0.3.1"], "autobuild": ["chromadb", "sentence-transformers", "huggingface-hub", "pysqlite3"], "teachable": ["chromadb"], "lmm": ["replicate", "pillow"], From 8e66590a1e07422e9550d62ddf557c457cc3f1b1 Mon Sep 17 00:00:00 2001 From: "F. Hinkelmann" Date: Tue, 2 Jul 2024 12:25:06 -0400 Subject: [PATCH 20/57] Fix simple typos in human-in-the-loop.ipynb (#3051) --- website/docs/tutorial/human-in-the-loop.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/docs/tutorial/human-in-the-loop.ipynb b/website/docs/tutorial/human-in-the-loop.ipynb index 04fbdd038b51..afcdeeaf42bf 100644 --- a/website/docs/tutorial/human-in-the-loop.ipynb +++ b/website/docs/tutorial/human-in-the-loop.ipynb @@ -10,7 +10,7 @@ "\n", "But many applications may require putting humans in-the-loop with agents. For example, to allow human feedback to steer agents in the right direction, specify goals, etc. In this chapter, we will show how AutoGen supports human intervention.\n", "\n", - "In AutoGen's `ConversableAgent`, the human-the-loop component sits in front\n", + "In AutoGen's `ConversableAgent`, the human-in-the-loop component sits in front\n", "of the auto-reply components. It can intercept the incoming messages and\n", "decide whether to pass them to the auto-reply components or to provide\n", "human feedback. The figure below illustrates the design.\n", @@ -285,9 +285,9 @@ "## Human Input Mode = `TERMINATE`\n", "\n", "In this mode, human input is only requested when a termination condition is\n", - "met. **If the human choose to intercept and reply, the counter will be reset**; if \n", - "the human choose to skip, automatic reply mechanism will be used; if the human\n", - "choose to terminate, the conversation will be terminated.\n", + "met. **If the human chooses to intercept and reply, the counter will be reset**; if \n", + "the human chooses to skip, the automatic reply mechanism will be used; if the human\n", + "chooses to terminate, the conversation will be terminated.\n", "\n", "Let us see this mode in action by playing the same game again, but this time\n", "the guessing agent will only have two chances to guess the number, and if it \n", From 3f1be8abcb4107b32dafe032f25f92bbe01dabee Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Tue, 2 Jul 2024 13:49:58 -0400 Subject: [PATCH 21/57] update readme (#3057) * update readme * Update README.md Co-authored-by: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> * add notion link --------- Co-authored-by: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> --- README.md | 7 ++++++- website/docs/Getting-Started.mdx | 11 ++++++----- website/docs/contributor-guide/contributing.md | 6 +++++- website/docusaurus.config.js | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5bff3300a50e..f14fe7a1a801 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,12 @@ ## What is AutoGen -AutoGen is a framework that enables the development of LLM applications using multiple agents that can converse with each other to solve tasks. AutoGen agents are customizable, conversable, and seamlessly allow human participation. They can operate in various modes that employ combinations of LLMs, human inputs, and tools. +AutoGen is an open-source programming framework for building AI agents and facilitating cooperation among multiple agents to solve tasks. AutoGen aims to streamline the development and research of agentic AI, much like PyTorch does for Deep Learning. It offers features such as agents capable of interacting with each other, facilitates the use of various large language models (LLMs) and tool use support, autonomous and human-in-the-loop workflows, and multi-agent conversation patterns. + +**Open Source Statement**: The project welcomes contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. Together, we can build something truly remarkable. + +The project is currently maintained by a [dynamic group of volunteers](https://butternut-swordtail-8a5.notion.site/410675be605442d3ada9a42eb4dfef30?v=fa5d0a79fd3d4c0f9c112951b2831cbb&pvs=4) from several different organizations. Contact project administrators Chi Wang and Qingyun Wu via auto-gen@outlook.com if you are interested in becoming a maintainer. + ![AutoGen Overview](https://github.com/microsoft/autogen/blob/main/website/static/img/autogen_agentchat.png) diff --git a/website/docs/Getting-Started.mdx b/website/docs/Getting-Started.mdx index 0f8c7322411d..3e162a098327 100644 --- a/website/docs/Getting-Started.mdx +++ b/website/docs/Getting-Started.mdx @@ -3,11 +3,12 @@ import TabItem from "@theme/TabItem"; # Getting Started -AutoGen is a framework that enables development of LLM applications using -multiple agents that can converse with each other to solve tasks. AutoGen agents -are customizable, conversable, and seamlessly allow human participation. They -can operate in various modes that employ combinations of LLMs, human inputs, and -tools. +AutoGen is an open-source programming framework for building AI agents and facilitating +cooperation among multiple agents to solve tasks. AutoGen aims to provide an easy-to-use +and flexible framework for accelerating development and research on agentic AI, +like PyTorch for Deep Learning. It offers features such as agents that can converse +with other agents, LLM and tool use support, autonomous and human-in-the-loop workflows, +and multi-agent conversation patterns. ![AutoGen Overview](/img/autogen_agentchat.png) diff --git a/website/docs/contributor-guide/contributing.md b/website/docs/contributor-guide/contributing.md index b90d81f227c8..b1b6b848f667 100644 --- a/website/docs/contributor-guide/contributing.md +++ b/website/docs/contributor-guide/contributing.md @@ -1,6 +1,6 @@ # Contributing to AutoGen -This project welcomes and encourages all forms of contributions, including but not limited to: +The project welcomes contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. Together, we can build something truly remarkable. Possible contributions include but not limited to: - Pushing patches. - Code review of pull requests. @@ -32,3 +32,7 @@ To see what we are working on and what we plan to work on, please check our ## Becoming a Reviewer There is currently no formal reviewer solicitation process. Current reviewers identify reviewers from active contributors. If you are willing to become a reviewer, you are welcome to let us know on discord. + +## Contact Maintainers + +The project is currently maintained by a [dynamic group of volunteers](https://butternut-swordtail-8a5.notion.site/410675be605442d3ada9a42eb4dfef30?v=fa5d0a79fd3d4c0f9c112951b2831cbb&pvs=4) from several different organizations. Contact project administrators Chi Wang and Qingyun Wu via auto-gen@outlook.com if you are interested in becoming a maintainer. diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index efc13096b0f7..0d581c83ddcd 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -14,7 +14,7 @@ customPostCssPlugin = () => { module.exports = { title: "AutoGen", - tagline: "Enable Next-Gen Large Language Model Applications", + tagline: "An Open-Source Programming Framework for Agentic AI", url: "https://microsoft.github.io", baseUrl: "/autogen/", onBrokenLinks: "throw", From fbf24a5f6f5a80f95650b5f6b7f1bee219350425 Mon Sep 17 00:00:00 2001 From: Mark Sze <66362098+marklysze@users.noreply.github.com> Date: Wed, 3 Jul 2024 05:28:24 +1000 Subject: [PATCH 22/57] Blog post for enhanced non-OpenAI model support (#2965) * Blogpost for enhanced non-OpenAI model support * update: quickstart with simple conversation * update: authors details * Added upfront text * Added function calling, refined text. Added chess for alt-models notebook, updated examples listing. * Added Groq to blog * Removed acknowledgements --------- Co-authored-by: Hk669 Co-authored-by: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> --- ...entchat_nested_chats_chess_altmodels.ipynb | 584 ++++++++++++++++++ .../img/agentstogether.jpeg | 3 + .../2024-06-24-AltModels-Classes/index.mdx | 393 ++++++++++++ website/blog/authors.yml | 12 + website/docs/Examples.md | 3 + website/docusaurus.config.js | 4 + 6 files changed, 999 insertions(+) create mode 100644 notebook/agentchat_nested_chats_chess_altmodels.ipynb create mode 100644 website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg create mode 100644 website/blog/2024-06-24-AltModels-Classes/index.mdx diff --git a/notebook/agentchat_nested_chats_chess_altmodels.ipynb b/notebook/agentchat_nested_chats_chess_altmodels.ipynb new file mode 100644 index 000000000000..69d3edbcfb50 --- /dev/null +++ b/notebook/agentchat_nested_chats_chess_altmodels.ipynb @@ -0,0 +1,584 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conversational Chess using non-OpenAI clients\n", + "\n", + "This notebook provides tips for using non-OpenAI models when using functions/tools.\n", + "\n", + "The code is based on [this notebook](/docs/notebooks/agentchat_nested_chats_chess),\n", + "which provides a detailed look at nested chats for tool use. Please refer to that\n", + "notebook for more on nested chats as this will be concentrated on tweaks to\n", + "improve performance with non-OpenAI models.\n", + "\n", + "The notebook represents a chess game between two players with a nested chat to\n", + "determine the available moves and select a move to make.\n", + "\n", + "This game contains a couple of functions/tools that the LLMs must use correctly by the\n", + "LLMs:\n", + "- `get_legal_moves` to get a list of current legal moves.\n", + "- `make_move` to make a move.\n", + "\n", + "Two agents will be used to represent the white and black players, each associated with\n", + "a different LLM cloud provider and model:\n", + "- Anthropic's Sonnet 3.5 will be Player_White\n", + "- Mistral's Mixtral 8x7B (using Together.AI) will be Player_Black\n", + "\n", + "As this involves function calling, we use larger, more capable, models from these providers.\n", + "\n", + "The nested chat will be supported be a board proxy agent who is set up to execute\n", + "the tools and manage the game.\n", + "\n", + "Tips to improve performance with these non-OpenAI models will be noted throughout **in bold**." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation\n", + "\n", + "First, you need to install the `pyautogen` and `chess` packages to use AutoGen. We'll include Anthropic and Together.AI libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install -qqq pyautogen[anthropic,together] chess" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up LLMs\n", + "\n", + "We'll use the Anthropic (`api_type` is `anthropic`) and Together.AI (`api_type` is `together`) client classes, with their respective models, which both support function calling." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import chess\n", + "import chess.svg\n", + "from IPython.display import display\n", + "from typing_extensions import Annotated\n", + "\n", + "from autogen import ConversableAgent, register_function\n", + "\n", + "# Let's set our two player configs, specifying clients and models\n", + "\n", + "# Anthropic's Sonnet for player white\n", + "player_white_config_list = [\n", + " {\n", + " \"api_type\": \"anthropic\",\n", + " \"model\": \"claude-3-5-sonnet-20240620\",\n", + " \"api_key\": os.getenv(\"ANTHROPIC_API_KEY\"),\n", + " \"cache_seed\": None,\n", + " },\n", + "]\n", + "\n", + "# Mistral's Mixtral 8x7B for player black (through Together.AI)\n", + "player_black_config_list = [\n", + " {\n", + " \"api_type\": \"together\",\n", + " \"model\": \"mistralai/Mixtral-8x7B-Instruct-v0.1\",\n", + " \"api_key\": os.environ.get(\"TOGETHER_API_KEY\"),\n", + " \"cache_seed\": None,\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll setup game variables and the two functions for getting the available moves and then making a move." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the board.\n", + "board = chess.Board()\n", + "\n", + "# Keep track of whether a move has been made.\n", + "made_move = False\n", + "\n", + "\n", + "def get_legal_moves() -> Annotated[\n", + " str,\n", + " \"Call this tool to list of all legal chess moves on the board, output is a list in UCI format, e.g. e2e4,e7e5,e7e8q.\",\n", + "]:\n", + " return \"Possible moves are: \" + \",\".join([str(move) for move in board.legal_moves])\n", + "\n", + "\n", + "def make_move(\n", + " move: Annotated[\n", + " str,\n", + " \"Call this tool to make a move after you have the list of legal moves and want to make a move. Takes UCI format, e.g. e2e4 or e7e5 or e7e8q.\",\n", + " ]\n", + ") -> Annotated[str, \"Result of the move.\"]:\n", + " move = chess.Move.from_uci(move)\n", + " board.push_uci(str(move))\n", + " global made_move\n", + " made_move = True\n", + " # Display the board.\n", + " display(\n", + " chess.svg.board(board, arrows=[(move.from_square, move.to_square)], fill={move.from_square: \"gray\"}, size=200)\n", + " )\n", + " # Get the piece name.\n", + " piece = board.piece_at(move.to_square)\n", + " piece_symbol = piece.unicode_symbol()\n", + " piece_name = (\n", + " chess.piece_name(piece.piece_type).capitalize()\n", + " if piece_symbol.isupper()\n", + " else chess.piece_name(piece.piece_type)\n", + " )\n", + " return f\"Moved {piece_name} ({piece_symbol}) from {chess.SQUARE_NAMES[move.from_square]} to {chess.SQUARE_NAMES[move.to_square]}.\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating agents\n", + "\n", + "Our main player agents are created next, with a few tweaks to help our models play:\n", + "\n", + "- Explicitly **telling agents their names** (as the name field isn't sent to the LLM).\n", + "- Providing simple instructions on the **order of functions** (not all models will need it).\n", + "- Asking the LLM to **include their name in the response** so the message content will include their names, helping the LLM understand who has made which moves.\n", + "- Ensure **no spaces are in the agent names** so that their name is distinguishable in the conversation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "player_white = ConversableAgent(\n", + " name=\"Player_White\",\n", + " system_message=\"You are a chess player and you play as white, your name is 'Player_White'. \"\n", + " \"First call the function get_legal_moves() to get list of legal moves. \"\n", + " \"Then call the function make_move(move) to make a move. \"\n", + " \"Then tell Player_Black you have made your move and it is their turn. \"\n", + " \"Make sure you tell Player_Black you are Player_White.\",\n", + " llm_config={\"config_list\": player_white_config_list, \"cache_seed\": None},\n", + ")\n", + "\n", + "player_black = ConversableAgent(\n", + " name=\"Player_Black\",\n", + " system_message=\"You are a chess player and you play as black, your name is 'Player_Black'. \"\n", + " \"First call the function get_legal_moves() to get list of legal moves. \"\n", + " \"Then call the function make_move(move) to make a move. \"\n", + " \"Then tell Player_White you have made your move and it is their turn. \"\n", + " \"Make sure you tell Player_White you are Player_Black.\",\n", + " llm_config={\"config_list\": player_black_config_list, \"cache_seed\": None},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we create a proxy agent that will be used to move the pieces on the board." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Check if the player has made a move, and reset the flag if move is made.\n", + "def check_made_move(msg):\n", + " global made_move\n", + " if made_move:\n", + " made_move = False\n", + " return True\n", + " else:\n", + " return False\n", + "\n", + "\n", + "board_proxy = ConversableAgent(\n", + " name=\"Board_Proxy\",\n", + " llm_config=False,\n", + " # The board proxy will only terminate the conversation if the player has made a move.\n", + " is_termination_msg=check_made_move,\n", + " # The auto reply message is set to keep the player agent retrying until a move is made.\n", + " default_auto_reply=\"Please make a move.\",\n", + " human_input_mode=\"NEVER\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our functions are then assigned to the agents so they can be passed to the LLM to choose from.\n", + "\n", + "We have tweaked the descriptions to provide **more guidance on when** to use it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "register_function(\n", + " make_move,\n", + " caller=player_white,\n", + " executor=board_proxy,\n", + " name=\"make_move\",\n", + " description=\"Call this tool to make a move after you have the list of legal moves.\",\n", + ")\n", + "\n", + "register_function(\n", + " get_legal_moves,\n", + " caller=player_white,\n", + " executor=board_proxy,\n", + " name=\"get_legal_moves\",\n", + " description=\"Call this to get a legal moves before making a move.\",\n", + ")\n", + "\n", + "register_function(\n", + " make_move,\n", + " caller=player_black,\n", + " executor=board_proxy,\n", + " name=\"make_move\",\n", + " description=\"Call this tool to make a move after you have the list of legal moves.\",\n", + ")\n", + "\n", + "register_function(\n", + " get_legal_moves,\n", + " caller=player_black,\n", + " executor=board_proxy,\n", + " name=\"get_legal_moves\",\n", + " description=\"Call this to get a legal moves before making a move.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Almost there, we now create nested chats between players and the board proxy agent to work out the available moves and make the move." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "player_white.register_nested_chats(\n", + " trigger=player_black,\n", + " chat_queue=[\n", + " {\n", + " # The initial message is the one received by the player agent from\n", + " # the other player agent.\n", + " \"sender\": board_proxy,\n", + " \"recipient\": player_white,\n", + " # The final message is sent to the player agent.\n", + " \"summary_method\": \"last_msg\",\n", + " }\n", + " ],\n", + ")\n", + "\n", + "player_black.register_nested_chats(\n", + " trigger=player_white,\n", + " chat_queue=[\n", + " {\n", + " # The initial message is the one received by the player agent from\n", + " # the other player agent.\n", + " \"sender\": board_proxy,\n", + " \"recipient\": player_black,\n", + " # The final message is sent to the player agent.\n", + " \"summary_method\": \"last_msg\",\n", + " }\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Playing the game\n", + "\n", + "Now the game can begin!" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mPlayer_Black\u001b[0m (to Player_White):\n", + "\n", + "Let's play chess! Your move.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[34mStarting a new chat....\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "Let's play chess! Your move.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Certainly! I'd be happy to play chess with you. As White, I'll make the first move. Let me start by checking the legal moves available to me.\n", + "\u001b[32m***** Suggested tool call (toolu_015sLMucefMVqS5ZNyWVGjgu): get_legal_moves *****\u001b[0m\n", + "Arguments: \n", + "{}\n", + "\u001b[32m*********************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_legal_moves...\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[32m***** Response from calling tool (toolu_015sLMucefMVqS5ZNyWVGjgu) *****\u001b[0m\n", + "Possible moves are: g1h3,g1f3,b1c3,b1a3,h2h3,g2g3,f2f3,e2e3,d2d3,c2c3,b2b3,a2a3,h2h4,g2g4,f2f4,e2e4,d2d4,c2c4,b2b4,a2a4\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Thank you for initiating a game of chess! As Player_White, I'll make the first move. After analyzing the legal moves, I've decided to make a classic opening move.\n", + "\u001b[32m***** Suggested tool call (toolu_01VjmBhHcGw5RTRKYC4Y5MeV): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\": \"e2e4\"}\n", + "\u001b[32m***************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION make_move...\u001b[0m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "
r n b q k b n r\n",
+       "p p p p p p p p\n",
+       ". . . . . . . .\n",
+       ". . . . . . . .\n",
+       ". . . . P . . .\n",
+       ". . . . . . . .\n",
+       "P P P P . P P P\n",
+       "R N B Q K B N R
" + ], + "text/plain": [ + "'
r n b q k b n r\\np p p p p p p p\\n. . . . . . . .\\n. . . . . . . .\\n. . . . P . . .\\n. . . . . . . .\\nP P P P . P P P\\nR N B Q K B N R
'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[32m***** Response from calling tool (toolu_01VjmBhHcGw5RTRKYC4Y5MeV) *****\u001b[0m\n", + "Moved pawn (โ™™) from e2 to e4.\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlayer_White\u001b[0m (to Player_Black):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[34mStarting a new chat....\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_z6jagiqn59m784w1n0zhmiop): get_legal_moves *****\u001b[0m\n", + "Arguments: \n", + "{}\n", + "\u001b[32m********************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_legal_moves...\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_z6jagiqn59m784w1n0zhmiop) *****\u001b[0m\n", + "Possible moves are: g8h6,g8f6,b8c6,b8a6,h7h6,g7g6,f7f6,e7e6,d7d6,c7c6,b7b6,a7a6,h7h5,g7g5,f7f5,e7e5,d7d5,c7c5,b7b5,a7a5\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_59t20pl0ab68z4xx2workgbc): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\":\"g8h6\"}\n", + "\u001b[32m**************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION make_move...\u001b[0m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "
r n b q k b . r\n",
+       "p p p p p p p p\n",
+       ". . . . . . . n\n",
+       ". . . . . . . .\n",
+       ". . . . P . . .\n",
+       ". . . . . . . .\n",
+       "P P P P . P P P\n",
+       "R N B Q K B N R
" + ], + "text/plain": [ + "'
r n b q k b . r\\np p p p p p p p\\n. . . . . . . n\\n. . . . . . . .\\n. . . . P . . .\\n. . . . . . . .\\nP P P P . P P P\\nR N B Q K B N R
'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_59t20pl0ab68z4xx2workgbc) *****\u001b[0m\n", + "Moved knight (โ™ž) from g8 to h6.\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_jwv1d86srs1fnvu33cky9tgv): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\":\"g8h6\"}\n", + "\u001b[32m**************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlayer_Black\u001b[0m (to Player_White):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n" + ] + } + ], + "source": [ + "# Clear the board.\n", + "board = chess.Board()\n", + "\n", + "chat_result = player_black.initiate_chat(\n", + " player_white,\n", + " message=\"Let's play chess! Your move.\",\n", + " max_turns=10,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At this stage, it's hard to tell who's going to win, but they're playing well and using the functions correctly." + ] + } + ], + "metadata": { + "front_matter": { + "description": "LLM-backed agents playing chess with each other using nested chats.", + "tags": [ + "nested chat", + "tool use", + "orchestration" + ] + }, + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg b/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg new file mode 100644 index 000000000000..fd859fc62076 --- /dev/null +++ b/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:964628601b60ddbab8940fea45014dcd841b89783bb0b7e9ac9d3690f1c41798 +size 659594 diff --git a/website/blog/2024-06-24-AltModels-Classes/index.mdx b/website/blog/2024-06-24-AltModels-Classes/index.mdx new file mode 100644 index 000000000000..9c94094e7e4c --- /dev/null +++ b/website/blog/2024-06-24-AltModels-Classes/index.mdx @@ -0,0 +1,393 @@ +--- +title: Enhanced Support for Non-OpenAI Models +authors: + - marklysze + - Hk669 +tags: [mistral ai,anthropic,together.ai,gemini] +--- + +![agents](img/agentstogether.jpeg) + +## TL;DR + +- **AutoGen has expanded integrations with a variety of cloud-based model providers beyond OpenAI.** +- **Leverage models and platforms from Gemini, Anthropic, Mistral AI, Together.AI, and Groq for your AutoGen agents.** +- **Utilise models specifically for chat, language, image, and coding.** +- **LLM provider diversification can provide cost and resilience benefits.** + +In addition to the recently released AutoGen [Google Gemini](https://ai.google.dev/) client, new client classes for [Mistral AI](https://mistral.ai/), [Anthropic](https://www.anthropic.com/), [Together.AI](https://www.together.ai/), and [Groq](https://groq.com/) enable you to utilize over 75 different large language models in your AutoGen agent workflow. + +These new client classes tailor AutoGen's underlying messages to each provider's unique requirements and remove that complexity from the developer, who can then focus on building their AutoGen workflow. + +Using them is as simple as installing the client-specific library and updating your LLM config with the relevant `api_type` and `model`. We'll demonstrate how to use them below. + +The community is continuing to enhance and build new client classes as cloud-based inference providers arrive. So, watch this space, and feel free to [discuss](https://discord.gg/pAbnFJrkgZ) or [develop](https://github.com/microsoft/autogen/pulls) another one. + +## Benefits of choice + +The need to use only the best models to overcome workflow-breaking LLM inconsistency has diminished considerably over the last 12 months. + +These new classes provide access to the very largest trillion-parameter models from OpenAI, Google, and Anthropic, continuing to provide the most consistent +and competent agent experiences. However, it's worth trying smaller models from the likes of Meta, Mistral AI, Microsoft, Qwen, and many others. Perhaps they +are capable enough for a task, or sub-task, or even better suited (such as a coding model)! + +Using smaller models will have cost benefits, but they also allow you to test models that you could run locally, allowing you to determine if you can remove cloud inference costs +altogether or even run an AutoGen workflow offline. + +On the topic of cost, these client classes also include provider-specific token cost calculations so you can monitor the cost impact of your workflows. With costs per million +tokens as low as 10 cents (and some are even free!), cost savings can be noticeable. + +## Mix and match + +How does Google's Gemini 1.5 Pro model stack up against Anthropic's Opus or Meta's Llama 3? + +Now you have the ability to quickly change your agent configs and find out. If you want to run all three in the one workflow, +AutoGen's ability to associate specific configurations to each agent means you can select the best LLM for each agent. + +## Capabilities + +The common requirements of text generation and function/tool calling are supported by these client classes. + +Multi-modal support, such as for image/audio/video, is an area of active development. The [Google Gemini](https://microsoft.github.io/autogen/docs/topics/non-openai-models/cloud-gemini) client class can be +used to create a multimodal agent. + +## Tips + +Here are some tips when working with these client classes: + +- **Most to least capable** - start with larger models and get your workflow working, then iteratively try smaller models. +- **Right model** - choose one that's suited to your task, whether it's coding, function calling, knowledge, or creative writing. +- **Agent names** - these cloud providers do not use the `name` field on a message, so be sure to use your agent's name in their `system_message` and `description` fields, as well as instructing the LLM to 'act as' them. This is particularly important for "auto" speaker selection in group chats as we need to guide the LLM to choose the next agent based on a name, so tweak `select_speaker_message_template`, `select_speaker_prompt_template`, and `select_speaker_auto_multiple_template` with more guidance. +- **Context length** - as your conversation gets longer, models need to support larger context lengths, be mindful of what the model supports and consider using [Transform Messages](https://microsoft.github.io/autogen/docs/topics/handling_long_contexts/intro_to_transform_messages) to manage context size. +- **Provider parameters** - providers have parameters you can set such as temperature, maximum tokens, top-k, top-p, and safety. See each client class in AutoGen's [API Reference](https://microsoft.github.io/autogen/docs/reference/oai/gemini) or [documentation](https://microsoft.github.io/autogen/docs/topics/non-openai-models/cloud-gemini) for details. +- **Prompts** - prompt engineering is critical in guiding smaller LLMs to do what you need. [ConversableAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/conversable_agent), [GroupChat](https://microsoft.github.io/autogen/docs/reference/agentchat/groupchat), [UserProxyAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/user_proxy_agent), and [AssistantAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/assistant_agent) all have customizable prompt attributes that you can tailor. Here are some prompting tips from [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/overview)([+Library](https://docs.anthropic.com/en/prompt-library/library)), [Mistral AI](https://docs.mistral.ai/guides/prompting_capabilities/), [Together.AI](https://docs.together.ai/docs/examples), and [Meta](https://llama.meta.com/docs/how-to-guides/prompting/). +- **Help!** - reach out on the AutoGen [Discord](https://discord.gg/pAbnFJrkgZ) or [log an issue](https://github.com/microsoft/autogen/issues) if you need help with or can help improve these client classes. + +Now it's time to try them out. + +## Quickstart + +### Installation + +Install the appropriate client based on the model you wish to use. + +```sh +pip install pyautogen["mistral"] # for Mistral AI client +pip install pyautogen["anthropic"] # for Anthropic client +pip install pyautogen["together"] # for Together.AI client +pip install pyautogen["groq"] # for Groq client +``` + +### Configuration Setup + +Add your model configurations to the `OAI_CONFIG_LIST`. Ensure you specify the `api_type` to initialize the respective client (Anthropic, Mistral, or Together). + +```yaml +[ + { + "model": "your anthropic model name", + "api_key": "your Anthropic api_key", + "api_type": "anthropic" + }, + { + "model": "your mistral model name", + "api_key": "your Mistral AI api_key", + "api_type": "mistral" + }, + { + "model": "your together.ai model name", + "api_key": "your Together.AI api_key", + "api_type": "together" + }, + { + "model": "your groq model name", + "api_key": "your Groq api_key", + "api_type": "groq" + } +] +``` + +### Usage + +The `[config_list_from_json](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils/#config_list_from_json)` function loads a list of configurations from an environment variable or a json file. + +```py +import autogen +from autogen import AssistantAgent, UserProxyAgent + +config_list = autogen.config_list_from_json( + "OAI_CONFIG_LIST" +) +``` + +### Construct Agents + +Construct a simple conversation between a User proxy and an Assistant agent + +```py +user_proxy = UserProxyAgent( + name="User_proxy", + code_execution_config={ + "last_n_messages": 2, + "work_dir": "groupchat", + "use_docker": False, # Please set use_docker = True if docker is available to run the generated code. Using docker is safer than running the generated code directly. + }, + human_input_mode="ALWAYS", + is_termination_msg=lambda msg: not msg["content"] +) + +assistant = AssistantAgent( + name="assistant", + llm_config = {"config_list": config_list} +) +``` + +### Start chat + +```py + +user_proxy.intiate_chat(assistant, message="Write python code to print Hello World!") + +``` + +**NOTE: To integrate this setup into GroupChat, follow the [tutorial](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat) with the same config as above.** + + +## Function Calls + +Now, let's look at how Anthropic's Sonnet 3.5 is able to suggest multiple function calls in a single response. + +This example is a simple travel agent setup with an agent for function calling and a user proxy agent for executing the functions. + +One thing you'll note here is Anthropic's models are more verbose than OpenAI's and will typically provide chain-of-thought or general verbiage when replying. Therefore we provide more explicit instructions to `functionbot` to not reply with more than necessary. Even so, it can't always help itself! + +Let's start with setting up our configuration and agents. + +```py +import os +import autogen +import json +from typing import Literal +from typing_extensions import Annotated + +# Anthropic configuration, using api_type='anthropic' +anthropic_llm_config = { + "config_list": + [ + { + "api_type": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "cache_seed": None + } + ] +} + +# Our functionbot, who will be assigned two functions and +# given directions to use them. +functionbot = autogen.AssistantAgent( + name="functionbot", + system_message="For currency exchange tasks, only use " + "the functions you have been provided with. Do not " + "reply with helpful tips. Once you've recommended functions " + "reply with 'TERMINATE'.", + is_termination_msg=lambda x: x.get("content", "") and (x.get("content", "").rstrip().endswith("TERMINATE") or x.get("content", "") == ""), + llm_config=anthropic_llm_config, +) + +# Our user proxy agent, who will be used to manage the customer +# request and conversation with the functionbot, terminating +# when we have the information we need. +user_proxy = autogen.UserProxyAgent( + name="user_proxy", + system_message="You are a travel agent that provides " + "specific information to your customers. Get the " + "information you need and provide a great summary " + "so your customer can have a great trip. If you " + "have the information you need, simply reply with " + "'TERMINATE'.", + is_termination_msg=lambda x: x.get("content", "") and (x.get("content", "").rstrip().endswith("TERMINATE") or x.get("content", "") == ""), + human_input_mode="NEVER", + max_consecutive_auto_reply=10, +) +``` + +We define the two functions. +```py +CurrencySymbol = Literal["USD", "EUR"] + +def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float: + if base_currency == quote_currency: + return 1.0 + elif base_currency == "USD" and quote_currency == "EUR": + return 1 / 1.1 + elif base_currency == "EUR" and quote_currency == "USD": + return 1.1 + else: + raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}") + +def get_current_weather(location, unit="fahrenheit"): + """Get the weather for some location""" + if "chicago" in location.lower(): + return json.dumps({"location": "Chicago", "temperature": "13", "unit": unit}) + elif "san francisco" in location.lower(): + return json.dumps({"location": "San Francisco", "temperature": "55", "unit": unit}) + elif "new york" in location.lower(): + return json.dumps({"location": "New York", "temperature": "11", "unit": unit}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) +``` + +And then associate them with the `user_proxy` for execution and `functionbot` for the LLM to consider using them. + +```py +@user_proxy.register_for_execution() +@functionbot.register_for_llm(description="Currency exchange calculator.") +def currency_calculator( + base_amount: Annotated[float, "Amount of currency in base_currency"], + base_currency: Annotated[CurrencySymbol, "Base currency"] = "USD", + quote_currency: Annotated[CurrencySymbol, "Quote currency"] = "EUR", +) -> str: + quote_amount = exchange_rate(base_currency, quote_currency) * base_amount + return f"{quote_amount} {quote_currency}" + +@user_proxy.register_for_execution() +@functionbot.register_for_llm(description="Weather forecast for US cities.") +def weather_forecast( + location: Annotated[str, "City name"], +) -> str: + weather_details = get_current_weather(location=location) + weather = json.loads(weather_details) + return f"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}" +``` + +Finally, we start the conversation with a request for help from our customer on their upcoming trip to New York and the Euro they would like exchanged to USD. + +Importantly, we're also using Anthropic's Sonnet to provide a summary through the `summary_method`. Using `summary_prompt`, we guide Sonnet to give us an email output. + +```py +# start the conversation +res = user_proxy.initiate_chat( + functionbot, + message="My customer wants to travel to New York and " + "they need to exchange 830 EUR to USD. Can you please " + "provide them with a summary of the weather and " + "exchanged currently in USD?", + summary_method="reflection_with_llm", + summary_args={ + "summary_prompt": """Summarize the conversation by + providing an email response with the travel information + for the customer addressed as 'Dear Customer'. Do not + provide any additional conversation or apologise, + just provide the relevant information and the email.""" + }, +) +``` + +After the conversation has finished, we'll print out the summary. + +```py +print(f"Here's the LLM summary of the conversation:\n\n{res.summary['content']}") +``` + +Here's the resulting output. + +```text +user_proxy (to functionbot): + +My customer wants to travel to New York and they need to exchange 830 EUR +to USD. Can you please provide them with a summary of the weather and +exchanged currently in USD? + +-------------------------------------------------------------------------------- +functionbot (to user_proxy): + +Certainly! I'd be happy to help your customer with information about the +weather in New York and the currency exchange from EUR to USD. Let's use +the available tools to get this information. + +***** Suggested tool call (toolu_016wBUKVX2TToBaMqmiGvhav): weather_forecast ***** +Arguments: +{"location": "New York"} +********************************************************************************** +***** Suggested tool call (toolu_01Nxjeew2BspfKdZ85on3XkP): currency_calculator ***** +Arguments: +{"base_amount": 830, "base_currency": "EUR", "quote_currency": "USD"} +************************************************************************************* + +-------------------------------------------------------------------------------- + +>>>>>>>> EXECUTING FUNCTION weather_forecast... + +>>>>>>>> EXECUTING FUNCTION currency_calculator... +user_proxy (to functionbot): + +user_proxy (to functionbot): + +***** Response from calling tool (toolu_016wBUKVX2TToBaMqmiGvhav) ***** +New York will be 11 degrees fahrenheit +*********************************************************************** + +-------------------------------------------------------------------------------- +user_proxy (to functionbot): + +***** Response from calling tool (toolu_01Nxjeew2BspfKdZ85on3XkP) ***** +913.0000000000001 USD +*********************************************************************** + +-------------------------------------------------------------------------------- +functionbot (to user_proxy): + +Thank you for providing the information. I can now give your customer a +summary of the weather in New York and the currency exchange from EUR to USD. + +Weather in New York: +The current forecast for New York indicates that it will be 11 degrees +Fahrenheit. This is quite cold, so your customer should be prepared with +warm clothing for their trip. + +Currency Exchange: +For 830 EUR, your customer will receive approximately 913 USD. This is based +on the current exchange rate provided by our currency calculator. + +To summarize: +1. Weather in New York: 11ยฐF (very cold) +2. Currency exchange: 830 EUR = 913 USD + +Your customer should pack warm clothes for the cold weather in New York and +can expect to have about 913 USD for their trip after exchanging 830 EUR. + +TERMINATE + +-------------------------------------------------------------------------------- +Here's the LLM summary of the conversation: + +Certainly. I'll provide an email response to the customer with the travel +information as requested. + +Dear Customer, + +We are pleased to provide you with the following information for your +upcoming trip to New York: + +Weather Forecast: +The current forecast for New York indicates a temperature of 11 degrees +Fahrenheit. Please be prepared for very cold weather and pack appropriate +warm clothing. + +Currency Exchange: +We have calculated the currency exchange for you. Your 830 EUR will be +equivalent to approximately 913 USD at the current exchange rate. + +We hope this information helps you prepare for your trip to New York. Have +a safe and enjoyable journey! + +Best regards, +Travel Assistance Team +``` + +So we can see how Anthropic's Sonnet is able to suggest multiple tools in a single response, with AutoGen executing them both and providing the results back to Sonnet. Sonnet then finishes with a nice email summary that can be the basis for continued real-life conversation with the customer. + +## More tips and tricks + +For an interesting chess game between Anthropic's Sonnet and Mistral's Mixtral, we've put together a sample notebook that highlights some of the tips and tricks for working with non-OpenAI LLMs. [See the notebook here](https://microsoft.github.io/autogen/docs/notebooks/agentchat_nested_chats_chess_altmodels). diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 83ea099ade48..0e023514465d 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -128,3 +128,15 @@ jluey: name: James Woffinden-Luey title: Senior Research Engineer at Microsoft Research url: https://github.com/jluey1 + +Hk669: + name: Hrushikesh Dokala + title: CS Undergraduate Based in India + url: https://github.com/Hk669 + image_url: https://github.com/Hk669.png + +marklysze: + name: Mark Sze + title: AI Freelancer + url: https://github.com/marklysze + image_url: https://github.com/marklysze.png diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 2ec83d1e0f21..3b71dd682e97 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -80,6 +80,9 @@ Links to notebook examples: - OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) - GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/gpt_assistant_agent_function_call.ipynb) +### Non-OpenAI Models +- Conversational Chess using non-OpenAI Models - [View Notebook](/docs/notebooks/agentchat_nested_chats_chess_altmodels) + ### Multimodal Agent - Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 0d581c83ddcd..f0c0f84a3945 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -281,6 +281,10 @@ module.exports = { to: "/docs/notebooks/agentchat_nested_chats_chess", from: ["/docs/notebooks/agentchat_chess"], }, + { + to: "/docs/notebooks/agentchat_nested_chats_chess_altmodels", + from: ["/docs/notebooks/agentchat_chess_altmodels"], + }, { to: "/docs/contributor-guide/contributing", from: ["/docs/Contribute"], From 8133b7de22a7b9a7c84cee31bbd963798efef753 Mon Sep 17 00:00:00 2001 From: "F. Hinkelmann" Date: Tue, 2 Jul 2024 22:26:07 -0400 Subject: [PATCH 23/57] Fix simple typo in chat-termination.ipynb (#3050) --- website/docs/tutorial/chat-termination.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/tutorial/chat-termination.ipynb b/website/docs/tutorial/chat-termination.ipynb index dcd5bdacecab..8da0aec6b92f 100644 --- a/website/docs/tutorial/chat-termination.ipynb +++ b/website/docs/tutorial/chat-termination.ipynb @@ -324,7 +324,7 @@ "You can configure both parameters in `initiate_chat` and also configuration of agents.\n", "\n", "That said, it is important to note that when a termination condition is triggered,\n", - "the conversation may not always terminated immediately. The actual termination\n", + "the conversation may not always terminate immediately. The actual termination\n", "depends on the `human_input_mode` argument of the `ConversableAgent` class.\n", "For example, when mode is `NEVER` the termination conditions above will end the conversations.\n", "But when mode is `ALWAYS` or `TERMINATE`, it will not terminate immediately.\n", From b4a3f263b0c20bbc1fac718f922636cf260d0a1a Mon Sep 17 00:00:00 2001 From: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> Date: Wed, 3 Jul 2024 20:03:03 +0530 Subject: [PATCH 24/57] Cohere Client (#3004) * initial setup for cohere client * client update * changes: ClintType added to the utils * Revert "changes: ClintType added to the utils" This reverts commit 80d61552287f2d2eff50b4c0f1a4adfd97233aa3. * Message conversion to Cohere, Parameter handling, cost calculation, streaming, tool calling * Changed Groq references. * minor fix * tests added * ref fix * added in the workflows * Fixed bug on non-streaming text generation * fix: formatting * Support Cohere rule for last message not USER when tool_results exist * Added Cohere to documentation * fixed client.py merge, removed unnecessary comments in groq.py, updated Cohere documentation, added Groq documentation * log: ignored params * update: custom exception added --------- Co-authored-by: Mark Sze Co-authored-by: Mark Sze <66362098+marklysze@users.noreply.github.com> --- .github/workflows/contrib-tests.yml | 36 ++ autogen/logger/file_logger.py | 12 +- autogen/logger/sqlite_logger.py | 12 +- autogen/oai/client.py | 12 + autogen/oai/cohere.py | 459 +++++++++++++++ autogen/oai/groq.py | 7 - autogen/runtime_logging.py | 5 +- setup.py | 1 + test/oai/test_cohere.py | 69 +++ .../non-openai-models/cloud-cohere.ipynb | 534 ++++++++++++++++++ .../topics/non-openai-models/cloud-groq.ipynb | 524 +++++++++++++++++ 11 files changed, 1661 insertions(+), 10 deletions(-) create mode 100644 autogen/oai/cohere.py create mode 100644 test/oai/test_cohere.py create mode 100644 website/docs/topics/non-openai-models/cloud-cohere.ipynb create mode 100644 website/docs/topics/non-openai-models/cloud-groq.ipynb diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index 895e810022de..c8d4d2672fa4 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -638,3 +638,39 @@ jobs: with: file: ./coverage.xml flags: unittests + + CohereTest: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for Cohere + run: | + pip install -e .[cohere,test] + - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash + run: | + if [[ ${{ matrix.os }} != ubuntu-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + - name: Coverage + run: | + pytest test/oai/test_cohere.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.py index cdebbdc0eb79..61a8a6335284 100644 --- a/autogen/logger/file_logger.py +++ b/autogen/logger/file_logger.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient + from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient @@ -205,7 +206,16 @@ def log_new_wrapper( def log_new_client( self, - client: AzureOpenAI | OpenAI | GeminiClient | AnthropicClient | MistralAIClient | TogetherClient | GroqClient, + client: ( + AzureOpenAI + | OpenAI + | GeminiClient + | AnthropicClient + | MistralAIClient + | TogetherClient + | GroqClient + | CohereClient + ), wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py index ccde6bd1d81b..2cf176ebb8f2 100644 --- a/autogen/logger/sqlite_logger.py +++ b/autogen/logger/sqlite_logger.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient + from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient @@ -392,7 +393,16 @@ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[st def log_new_client( self, - client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient], + client: Union[ + AzureOpenAI, + OpenAI, + GeminiClient, + AnthropicClient, + MistralAIClient, + TogetherClient, + GroqClient, + CohereClient, + ], wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/oai/client.py b/autogen/oai/client.py index a7b12ce83dad..ef3a3fd2b1b3 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -77,6 +77,13 @@ except ImportError as e: groq_import_exception = e +try: + from autogen.oai.cohere import CohereClient + + cohere_import_exception: Optional[ImportError] = None +except ImportError as e: + cohere_import_exception = e + logger = logging.getLogger(__name__) if not logger.handlers: # Add the console handler. @@ -497,6 +504,11 @@ def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[s raise ImportError("Please install `groq` to use the Groq API.") client = GroqClient(**openai_config) self._clients.append(client) + elif api_type is not None and api_type.startswith("cohere"): + if cohere_import_exception: + raise ImportError("Please install `cohere` to use the Groq API.") + client = CohereClient(**openai_config) + self._clients.append(client) else: client = OpenAI(**openai_config) self._clients.append(OpenAIClient(client)) diff --git a/autogen/oai/cohere.py b/autogen/oai/cohere.py new file mode 100644 index 000000000000..e04d07327203 --- /dev/null +++ b/autogen/oai/cohere.py @@ -0,0 +1,459 @@ +"""Create an OpenAI-compatible client using Cohere's API. + +Example: + llm_config={ + "config_list": [{ + "api_type": "cohere", + "model": "command-r-plus", + "api_key": os.environ.get("COHERE_API_KEY") + } + ]} + + agent = autogen.AssistantAgent("my_agent", llm_config=llm_config) + +Install Cohere's python library using: pip install --upgrade cohere + +Resources: +- https://docs.cohere.com/reference/chat +""" + +from __future__ import annotations + +import json +import logging +import os +import random +import sys +import time +import warnings +from typing import Any, Dict, List + +from cohere import Client as Cohere +from cohere.types import ToolParameterDefinitionsValue, ToolResult +from flaml.automl.logger import logger_formatter +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.completion_usage import CompletionUsage + +from autogen.oai.client_utils import validate_parameter + +logger = logging.getLogger(__name__) +if not logger.handlers: + # Add the console handler. + _ch = logging.StreamHandler(stream=sys.stdout) + _ch.setFormatter(logger_formatter) + logger.addHandler(_ch) + + +COHERE_PRICING_1K = { + "command-r-plus": (0.003, 0.015), + "command-r": (0.0005, 0.0015), + "command-nightly": (0.00025, 0.00125), + "command": (0.015, 0.075), + "command-light": (0.008, 0.024), + "command-light-nightly": (0.008, 0.024), +} + + +class CohereClient: + """Client for Cohere's API.""" + + def __init__(self, **kwargs): + """Requires api_key or environment variable to be set + + Args: + api_key (str): The API key for using Cohere (or environment variable COHERE_API_KEY needs to be set) + """ + # Ensure we have the api_key upon instantiation + self.api_key = kwargs.get("api_key", None) + if not self.api_key: + self.api_key = os.getenv("COHERE_API_KEY") + + assert ( + self.api_key + ), "Please include the api_key in your config list entry for Cohere or set the COHERE_API_KEY env variable." + + def message_retrieval(self, response) -> List: + """ + Retrieve and return a list of strings or a list of Choice.Message from the response. + + NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object, + since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used. + """ + return [choice.message for choice in response.choices] + + def cost(self, response) -> float: + return response.cost + + @staticmethod + def get_usage(response) -> Dict: + """Return usage summary of the response using RESPONSE_USAGE_KEYS.""" + # ... # pragma: no cover + return { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "cost": response.cost, + "model": response.model, + } + + def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Loads the parameters for Cohere API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults""" + cohere_params = {} + + # Check that we have what we need to use Cohere's API + # We won't enforce the available models as they are likely to change + cohere_params["model"] = params.get("model", None) + assert cohere_params[ + "model" + ], "Please specify the 'model' in your config list entry to nominate the Cohere model to use." + + # Validate allowed Cohere parameters + # https://docs.cohere.com/reference/chat + cohere_params["temperature"] = validate_parameter( + params, "temperature", (int, float), False, 0.3, (0, None), None + ) + cohere_params["max_tokens"] = validate_parameter(params, "max_tokens", int, True, None, (0, None), None) + cohere_params["k"] = validate_parameter(params, "k", int, False, 0, (0, 500), None) + cohere_params["p"] = validate_parameter(params, "p", (int, float), False, 0.75, (0.01, 0.99), None) + cohere_params["seed"] = validate_parameter(params, "seed", int, True, None, None, None) + cohere_params["frequency_penalty"] = validate_parameter( + params, "frequency_penalty", (int, float), True, 0, (0, 1), None + ) + cohere_params["presence_penalty"] = validate_parameter( + params, "presence_penalty", (int, float), True, 0, (0, 1), None + ) + + # Cohere parameters we are ignoring: + # preamble - we will put the system prompt in here. + # parallel_tool_calls (defaults to True), perfect as is. + # conversation_id - allows resuming a previous conversation, we don't support this. + logging.info("Conversation ID: %s", params.get("conversation_id", "None")) + # connectors - allows web search or other custom connectors, not implementing for now but could be useful in the future. + logging.info("Connectors: %s", params.get("connectors", "None")) + # search_queries_only - to control whether only search queries are used, we're not using connectors so ignoring. + # documents - a list of documents that can be used to support the chat. Perhaps useful in the future for RAG. + # citation_quality - used for RAG flows and dependent on other parameters we're ignoring. + # max_input_tokens - limits input tokens, not needed. + logging.info("Max Input Tokens: %s", params.get("max_input_tokens", "None")) + # stop_sequences - used to stop generation, not needed. + logging.info("Stop Sequences: %s", params.get("stop_sequences", "None")) + + return cohere_params + + def create(self, params: Dict) -> ChatCompletion: + + messages = params.get("messages", []) + + # Parse parameters to the Cohere API's parameters + cohere_params = self.parse_params(params) + + # Convert AutoGen messages to Cohere messages + cohere_messages, preamble, final_message = oai_messages_to_cohere_messages(messages, params, cohere_params) + + cohere_params["chat_history"] = cohere_messages + cohere_params["message"] = final_message + cohere_params["preamble"] = preamble + + # We use chat model by default + client = Cohere(api_key=self.api_key) + + # Token counts will be returned + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + # Stream if in parameters + streaming = True if "stream" in params and params["stream"] else False + cohere_finish = "" + + max_retries = 5 + for attempt in range(max_retries): + ans = None + try: + if streaming: + response = client.chat_stream(**cohere_params) + else: + response = client.chat(**cohere_params) + except CohereRateLimitError as e: + raise RuntimeError(f"Cohere exception occurred: {e}") + else: + + if streaming: + # Streaming... + ans = "" + for event in response: + if event.event_type == "text-generation": + ans = ans + event.text + elif event.event_type == "tool-calls-generation": + # When streaming, tool calls are compiled at the end into a single event_type + ans = event.text + cohere_finish = "tool_calls" + tool_calls = [] + for tool_call in event.tool_calls: + tool_calls.append( + ChatCompletionMessageToolCall( + id=str(random.randint(0, 100000)), + function={ + "name": tool_call.name, + "arguments": ( + "" if tool_call.parameters is None else json.dumps(tool_call.parameters) + ), + }, + type="function", + ) + ) + + # Not using billed_units, but that may be better for cost purposes + prompt_tokens = event.response.meta.tokens.input_tokens + completion_tokens = event.response.meta.tokens.output_tokens + total_tokens = prompt_tokens + completion_tokens + + response_id = event.response.response_id + else: + # Non-streaming finished + ans: str = response.text + + # Not using billed_units, but that may be better for cost purposes + prompt_tokens = response.meta.tokens.input_tokens + completion_tokens = response.meta.tokens.output_tokens + total_tokens = prompt_tokens + completion_tokens + + response_id = response.response_id + break + + if response is not None: + + response_content = ans + + if streaming: + # Streaming response + if cohere_finish == "": + cohere_finish = "stop" + tool_calls = None + else: + # Non-streaming response + # If we have tool calls as the response, populate completed tool calls for our return OAI response + if response.tool_calls is not None: + cohere_finish = "tool_calls" + tool_calls = [] + for tool_call in response.tool_calls: + + # if parameters are null, clear them out (Cohere can return a string "null" if no parameter values) + + tool_calls.append( + ChatCompletionMessageToolCall( + id=str(random.randint(0, 100000)), + function={ + "name": tool_call.name, + "arguments": ( + "" if tool_call.parameters is None else json.dumps(tool_call.parameters) + ), + }, + type="function", + ) + ) + else: + cohere_finish = "stop" + tool_calls = None + else: + raise RuntimeError(f"Failed to get response from Cohere after retrying {attempt + 1} times.") + + # 3. convert output + message = ChatCompletionMessage( + role="assistant", + content=response_content, + function_call=None, + tool_calls=tool_calls, + ) + choices = [Choice(finish_reason=cohere_finish, index=0, message=message)] + + response_oai = ChatCompletion( + id=response_id, + model=cohere_params["model"], + created=int(time.time()), + object="chat.completion", + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + cost=calculate_cohere_cost(prompt_tokens, completion_tokens, cohere_params["model"]), + ) + + return response_oai + + +def oai_messages_to_cohere_messages( + messages: list[Dict[str, Any]], params: Dict[str, Any], cohere_params: Dict[str, Any] +) -> tuple[list[dict[str, Any]], str, str]: + """Convert messages from OAI format to Cohere's format. + We correct for any specific role orders and types. + + Parameters: + messages: list[Dict[str, Any]]: AutoGen messages + params: Dict[str, Any]: AutoGen parameters dictionary + cohere_params: Dict[str, Any]: Cohere parameters dictionary + + Returns: + List[Dict[str, Any]]: Chat History messages + str: Preamble (system message) + str: Message (the final user message) + """ + + cohere_messages = [] + preamble = "" + + # Tools + if "tools" in params: + cohere_tools = [] + for tool in params["tools"]: + + # build list of properties + parameters = {} + + for key, value in tool["function"]["parameters"]["properties"].items(): + type_str = value["type"] + required = True # Defaults to False, we could consider leaving it as default. + description = value["description"] + + # If we have an 'enum' key, add that to the description (as not allowed to pass in enum as a field) + if "enum" in value: + # Access the enum list + enum_values = value["enum"] + enum_strings = [str(value) for value in enum_values] + enum_string = ", ".join(enum_strings) + description = description + ". Possible values are " + enum_string + "." + + parameters[key] = ToolParameterDefinitionsValue( + description=description, type=type_str, required=required + ) + + cohere_tool = { + "name": tool["function"]["name"], + "description": tool["function"]["description"], + "parameter_definitions": parameters, + } + + cohere_tools.append(cohere_tool) + + if len(cohere_tools) > 0: + cohere_params["tools"] = cohere_tools + + tool_calls = [] + tool_results = [] + + # Rules for cohere messages: + # no 'name' field + # 'system' messages go into the preamble parameter + # user role = 'USER' + # assistant role = 'CHATBOT' + # 'content' field renamed to 'message' + # tools go into tools parameter + # tool_results go into tool_results parameter + for message in messages: + + if "role" in message and message["role"] == "system": + # System message + if preamble == "": + preamble = message["content"] + else: + preamble = preamble + "\n" + message["content"] + elif "tool_calls" in message: + # Suggested tool calls, build up the list before we put it into the tool_results + for tool_call in message["tool_calls"]: + tool_calls.append(tool_call) + + # We also add the suggested tool call as a message + new_message = { + "role": "CHATBOT", + "message": message["content"], + # Not including tools in this message, may need to. Testing required. + } + + cohere_messages.append(new_message) + elif "role" in message and message["role"] == "tool": + if "tool_call_id" in message: + # Convert the tool call to a result + + tool_call_id = message["tool_call_id"] + content_output = message["content"] + + # Find the original tool + for tool_call in tool_calls: + if tool_call["id"] == tool_call_id: + + call = { + "name": tool_call["function"]["name"], + "parameters": json.loads( + tool_call["function"]["arguments"] + if not tool_call["function"]["arguments"] == "" + else "{}" + ), + } + output = [{"value": content_output}] + + tool_results.append(ToolResult(call=call, outputs=output)) + + break + elif "content" in message and isinstance(message["content"], str): + # Standard text message + new_message = { + "role": "USER" if message["role"] == "user" else "CHATBOT", + "message": message["content"], + } + + cohere_messages.append(new_message) + + # Append any Tool Results + if len(tool_results) != 0: + cohere_params["tool_results"] = tool_results + + # Enable multi-step tool use: https://docs.cohere.com/docs/multi-step-tool-use + cohere_params["force_single_step"] = False + + # If we're adding tool_results, like we are, the last message can't be a USER message + # So, we add a CHATBOT 'continue' message, if so. + if cohere_messages[-1]["role"] == "USER": + cohere_messages.append({"role": "CHATBOT", "content": "Please continue."}) + + # We return a blank message when we have tool results + # TODO: Check what happens if tool_results aren't the latest message + return cohere_messages, preamble, "" + + else: + + # We need to get the last message to assign to the message field for Cohere, + # if the last message is a user message, use that, otherwise put in 'continue'. + if cohere_messages[-1]["role"] == "USER": + return cohere_messages[0:-1], preamble, cohere_messages[-1]["message"] + else: + return cohere_messages, preamble, "Please continue." + + +def calculate_cohere_cost(input_tokens: int, output_tokens: int, model: str) -> float: + """Calculate the cost of the completion using the Cohere pricing.""" + total = 0.0 + + if model in COHERE_PRICING_1K: + input_cost_per_k, output_cost_per_k = COHERE_PRICING_1K[model] + input_cost = (input_tokens / 1000) * input_cost_per_k + output_cost = (output_tokens / 1000) * output_cost_per_k + total = input_cost + output_cost + else: + warnings.warn(f"Cost calculation not available for {model} model", UserWarning) + + return total + + +class CohereError(Exception): + """Base class for other Cohere exceptions""" + + pass + + +class CohereRateLimitError(CohereError): + """Raised when rate limit is exceeded""" + + pass diff --git a/autogen/oai/groq.py b/autogen/oai/groq.py index a97240887c8e..d2abe5116a25 100644 --- a/autogen/oai/groq.py +++ b/autogen/oai/groq.py @@ -259,13 +259,6 @@ def oai_messages_to_groq_messages(messages: list[Dict[str, Any]]) -> list[dict[s groq_messages = copy.deepcopy(messages) - # If we have a message with role='tool', which occurs when a function is executed, change it to 'user' - """ - for msg in together_messages: - if "role" in msg and msg["role"] == "tool": - msg["role"] = "user" - """ - # Remove the name field for message in groq_messages: if "name" in message: diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py index 4ad76cf5b7da..1ffc8b622f0a 100644 --- a/autogen/runtime_logging.py +++ b/autogen/runtime_logging.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient + from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient @@ -111,7 +112,9 @@ def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig def log_new_client( - client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient], + client: Union[ + AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient, CohereClient + ], wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/setup.py b/setup.py index cfaad885968c..9117ed45ceac 100644 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ "anthropic": ["anthropic>=0.23.1"], "mistral": ["mistralai>=0.2.0"], "groq": ["groq>=0.9.0"], + "cohere": ["cohere>=5.5.8"], } setuptools.setup( diff --git a/test/oai/test_cohere.py b/test/oai/test_cohere.py new file mode 100644 index 000000000000..83ef56b17087 --- /dev/null +++ b/test/oai/test_cohere.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 -m pytest + +import os + +import pytest + +try: + from autogen.oai.cohere import CohereClient, calculate_cohere_cost + + skip = False +except ImportError: + CohereClient = object + skip = True + + +reason = "Cohere dependency not installed!" + + +@pytest.fixture() +def cohere_client(): + return CohereClient(api_key="dummy_api_key") + + +@pytest.mark.skipif(skip, reason=reason) +def test_initialization_missing_api_key(): + os.environ.pop("COHERE_API_KEY", None) + with pytest.raises( + AssertionError, + match="Please include the api_key in your config list entry for Cohere or set the COHERE_API_KEY env variable.", + ): + CohereClient() + + CohereClient(api_key="dummy_api_key") + + +@pytest.mark.skipif(skip, reason=reason) +def test_intialization(cohere_client): + assert cohere_client.api_key == "dummy_api_key", "`api_key` should be correctly set in the config" + + +@pytest.mark.skipif(skip, reason=reason) +def test_calculate_cohere_cost(): + assert ( + calculate_cohere_cost(0, 0, model="command-r") == 0.0 + ), "Cost should be 0 for 0 input_tokens and 0 output_tokens" + assert calculate_cohere_cost(100, 200, model="command-r-plus") == 0.0033 + + +@pytest.mark.skipif(skip, reason=reason) +def test_load_config(cohere_client): + params = { + "model": "command-r-plus", + "stream": False, + "temperature": 1, + "p": 0.8, + "max_tokens": 100, + } + expected_params = { + "model": "command-r-plus", + "temperature": 1, + "p": 0.8, + "seed": None, + "max_tokens": 100, + "frequency_penalty": 0, + "presence_penalty": 0, + "k": 0, + } + result = cohere_client.parse_params(params) + assert result == expected_params, "Config should be correctly loaded" diff --git a/website/docs/topics/non-openai-models/cloud-cohere.ipynb b/website/docs/topics/non-openai-models/cloud-cohere.ipynb new file mode 100644 index 000000000000..fed5911475f4 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-cohere.ipynb @@ -0,0 +1,534 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cohere\n", + "\n", + "[Cohere](https://cohere.com/) is a cloud based platform serving their own LLMs, in particular the Command family of models.\n", + "\n", + "Cohere's API differs from OpenAI's, which is the native API used by AutoGen, so to use Cohere's LLMs you need to use this library.\n", + "\n", + "You will need a Cohere account and create an API key. [See their website for further details](https://cohere.com/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Features\n", + "\n", + "When using this client class, AutoGen's messages are automatically tailored to accommodate the specific requirements of Cohere's API.\n", + "\n", + "Additionally, this client class provides support for function/tool calling and will track token usage and cost correctly as per Cohere's API costs (as of July 2024).\n", + "\n", + "## Getting started\n", + "\n", + "First you need to install the `pyautogen` package to use AutoGen with the Cohere API library.\n", + "\n", + "``` bash\n", + "pip install pyautogen[cohere]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Cohere provides a number of models to use, included below. See the list of [models here](https://docs.cohere.com/docs/models).\n", + "\n", + "See the sample `OAI_CONFIG_LIST` below showing how the Cohere client class is used by specifying the `api_type` as `cohere`.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"gpt-35-turbo\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"gpt-4-vision-preview\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"dalle\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"command-r-plus\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\"\n", + " },\n", + " {\n", + " \"model\": \"command-r\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\"\n", + " },\n", + " {\n", + " \"model\": \"command\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\"\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an alternative to the `api_key` key and value in the config, you can set the environment variable `COHERE_API_KEY` to your Cohere key.\n", + "\n", + "Linux/Mac:\n", + "``` bash\n", + "export COHERE_API_KEY=\"your_cohere_api_key_here\"\n", + "```\n", + "\n", + "Windows:\n", + "``` bash\n", + "set COHERE_API_KEY=your_cohere_api_key_here\n", + "```\n", + "\n", + "## API parameters\n", + "\n", + "The following parameters can be added to your config for the Cohere API. See [this link](https://docs.cohere.com/reference/chat) for further information on them and their default values.\n", + "\n", + "- temperature (number > 0)\n", + "- p (number 0.01..0.99)\n", + "- k (number 0..500)\n", + "- max_tokens (null, integer >= 0)\n", + "- seed (null, integer)\n", + "- frequency_penalty (number 0..1)\n", + "- presence_penalty (number 0..1)\n", + "\n", + "Example:\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"command-r\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\",\n", + " \"temperature\": 0.5,\n", + " \"p\": 0.2,\n", + " \"k\": 100,\n", + " \"max_tokens\": 2048,\n", + " \"seed\": 42,\n", + " \"frequency_penalty\": 0.5,\n", + " \"presence_penalty\": 0.2\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Two-Agent Coding Example\n", + "\n", + "In this example, we run a two-agent chat with an AssistantAgent (primarily a coding agent) to generate code to count the number of prime numbers between 1 and 10,000 and then it will be executed.\n", + "\n", + "We'll use Cohere's Command R model which is suitable for coding." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "config_list = [\n", + " {\n", + " # Let's choose the Command-R model\n", + " \"model\": \"command-r\",\n", + " # Provide your Cohere's API key here or put it into the COHERE_API_KEY environment variable.\n", + " \"api_key\": os.environ.get(\"COHERE_API_KEY\"),\n", + " # We specify the API Type as 'cohere' so it uses the Cohere client class\n", + " \"api_type\": \"cohere\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, we have tweaked the system message so that the model doesn't return the termination keyword, which we've changed to FINISH, with the code block." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.coding import LocalCommandLineCodeExecutor\n", + "\n", + "# Setting up the code executor\n", + "workdir = Path(\"coding\")\n", + "workdir.mkdir(exist_ok=True)\n", + "code_executor = LocalCommandLineCodeExecutor(work_dir=workdir)\n", + "\n", + "# Setting up the agents\n", + "\n", + "# The UserProxyAgent will execute the code that the AssistantAgent provides\n", + "user_proxy_agent = UserProxyAgent(\n", + " name=\"User\",\n", + " code_execution_config={\"executor\": code_executor},\n", + " is_termination_msg=lambda msg: \"FINISH\" in msg.get(\"content\"),\n", + ")\n", + "\n", + "system_message = \"\"\"You are a helpful AI assistant who writes code and the user executes it.\n", + "Solve tasks using your coding and language skills.\n", + "In the following cases, suggest python code (in a python coding block) for the user to execute.\n", + "Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.\n", + "When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.\n", + "Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.\n", + "If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.\n", + "When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.\n", + "IMPORTANT: Wait for the user to execute your code and then you can reply with the word \"FINISH\". DO NOT OUTPUT \"FINISH\" after your code block.\"\"\"\n", + "\n", + "# The AssistantAgent, using Cohere's model, will take the coding request and return code\n", + "assistant_agent = AssistantAgent(\n", + " name=\"Cohere Assistant\",\n", + " system_message=system_message,\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to Cohere Assistant):\n", + "\n", + "Provide code to count the number of prime numbers from 1 to 10000.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mCohere Assistant\u001b[0m (to User):\n", + "\n", + "Here's the code to count the number of prime numbers from 1 to 10,000:\n", + "```python\n", + "# Prime Number Counter\n", + "count = 0\n", + "for num in range(2, 10001):\n", + " if num > 1:\n", + " for div in range(2, num):\n", + " if (num % div) == 0:\n", + " break\n", + " else:\n", + " count += 1\n", + "print(count)\n", + "```\n", + "\n", + "My plan is to use two nested loops. The outer loop iterates through numbers from 2 to 10,000. The inner loop checks if there's any divisor for the current number in the range from 2 to the number itself. If there's no such divisor, the number is prime and the counter is incremented.\n", + "\n", + "Please execute the code and let me know the output.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Cohere Assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: 1229\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mCohere Assistant\u001b[0m (to User):\n", + "\n", + "That's correct! The code you executed successfully found 1229 prime numbers within the specified range.\n", + "\n", + "FINISH.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "# Start the chat, with the UserProxyAgent asking the AssistantAgent the message\n", + "chat_result = user_proxy_agent.initiate_chat(\n", + " assistant_agent,\n", + " message=\"Provide code to count the number of prime numbers from 1 to 10000.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Call Example\n", + "\n", + "In this example, instead of writing code, we will show how Cohere's Command R+ model can perform parallel tool calling, where it recommends calling more than one tool at a time.\n", + "\n", + "We'll use a simple travel agent assistant program where we have a couple of tools for weather and currency conversion.\n", + "\n", + "We start by importing libraries and setting up our configuration to use Command R+ and the `cohere` client class." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import Literal\n", + "\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "\n", + "config_list = [\n", + " {\"api_type\": \"cohere\", \"model\": \"command-r-plus\", \"api_key\": os.getenv(\"COHERE_API_KEY\"), \"cache_seed\": None}\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create our two agents." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the agent for tool calling\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"\"\"For currency exchange and weather forecasting tasks,\n", + " only use the functions you have been provided with.\n", + " Output 'HAVE FUN!' when an answer has been provided.\"\"\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# Note that we have changed the termination string to be \"HAVE FUN!\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"HAVE FUN!\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the two functions, annotating them so that those descriptions can be passed through to the LLM.\n", + "\n", + "We associate them with the agents using `register_for_execution` for the user_proxy so it can execute the function and `register_for_llm` for the chatbot (powered by the LLM) so it can pass the function definitions to the LLM." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Currency Exchange function\n", + "\n", + "CurrencySymbol = Literal[\"USD\", \"EUR\"]\n", + "\n", + "# Define our function that we expect to call\n", + "\n", + "\n", + "def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float:\n", + " if base_currency == quote_currency:\n", + " return 1.0\n", + " elif base_currency == \"USD\" and quote_currency == \"EUR\":\n", + " return 1 / 1.1\n", + " elif base_currency == \"EUR\" and quote_currency == \"USD\":\n", + " return 1.1\n", + " else:\n", + " raise ValueError(f\"Unknown currencies {base_currency}, {quote_currency}\")\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Currency exchange calculator.\")\n", + "def currency_calculator(\n", + " base_amount: Annotated[float, \"Amount of currency in base_currency\"],\n", + " base_currency: Annotated[CurrencySymbol, \"Base currency\"] = \"USD\",\n", + " quote_currency: Annotated[CurrencySymbol, \"Quote currency\"] = \"EUR\",\n", + ") -> str:\n", + " quote_amount = exchange_rate(base_currency, quote_currency) * base_amount\n", + " return f\"{format(quote_amount, '.2f')} {quote_currency}\"\n", + "\n", + "\n", + "# Weather function\n", + "\n", + "\n", + "# Example function to make available to model\n", + "def get_current_weather(location, unit=\"fahrenheit\"):\n", + " \"\"\"Get the weather for some location\"\"\"\n", + " if \"chicago\" in location.lower():\n", + " return json.dumps({\"location\": \"Chicago\", \"temperature\": \"13\", \"unit\": unit})\n", + " elif \"san francisco\" in location.lower():\n", + " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"55\", \"unit\": unit})\n", + " elif \"new york\" in location.lower():\n", + " return json.dumps({\"location\": \"New York\", \"temperature\": \"11\", \"unit\": unit})\n", + " else:\n", + " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Weather forecast for US cities.\")\n", + "def weather_forecast(\n", + " location: Annotated[str, \"City name\"],\n", + ") -> str:\n", + " weather_details = get_current_weather(location=location)\n", + " weather = json.loads(weather_details)\n", + " return f\"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We pass through our customers message and run the chat.\n", + "\n", + "Finally, we ask the LLM to summarise the chat and print that out." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "I will use the weather_forecast function to find out the weather in New York, and the currency_calculator function to convert 123.45 EUR to USD. I will then search for 'holiday tips' to find some extra information to include in my answer.\n", + "\u001b[32m***** Suggested tool call (45212): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\": \"New York\"}\n", + "\u001b[32m*********************************************************\u001b[0m\n", + "\u001b[32m***** Suggested tool call (16564): currency_calculator *****\u001b[0m\n", + "Arguments: \n", + "{\"base_amount\": 123.45, \"base_currency\": \"EUR\", \"quote_currency\": \"USD\"}\n", + "\u001b[32m************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION weather_forecast...\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION currency_calculator...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (45212) *****\u001b[0m\n", + "New York will be 11 degrees fahrenheit\n", + "\u001b[32m**********************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (16564) *****\u001b[0m\n", + "135.80 USD\n", + "\u001b[32m**********************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "The weather in New York is 11 degrees Fahrenheit. \n", + "\n", + "โ‚ฌ123.45 is worth $135.80. \n", + "\n", + "Here are some holiday tips:\n", + "- Make sure to pack layers for the cold weather\n", + "- Try the local cuisine, New York is famous for its pizza\n", + "- Visit Central Park and take in the views from the top of the Rockefeller Centre\n", + "\n", + "HAVE FUN!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "LLM SUMMARY: The weather in New York is 11 degrees Fahrenheit. 123.45 EUR is worth 135.80 USD. Holiday tips: make sure to pack warm clothes and have a great time!\n" + ] + } + ], + "source": [ + "# start the conversation\n", + "res = user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\",\n", + " summary_method=\"reflection_with_llm\",\n", + ")\n", + "\n", + "print(f\"LLM SUMMARY: {res.summary['content']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that Command R+ recommended we call both tools and passed through the right parameters. The `user_proxy` executed them and this was passed back to Command R+ to interpret them and respond. Finally, Command R+ was asked to summarise the whole conversation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/website/docs/topics/non-openai-models/cloud-groq.ipynb b/website/docs/topics/non-openai-models/cloud-groq.ipynb new file mode 100644 index 000000000000..d2289cbdcd45 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-groq.ipynb @@ -0,0 +1,524 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Groq\n", + "\n", + "[Groq](https://groq.com/) is a cloud based platform serving a number of popular open weight models at high inference speeds. Models include Meta's Llama 3, Mistral AI's Mixtral, and Google's Gemma.\n", + "\n", + "Although Groq's API is aligned well with OpenAI's, which is the native API used by AutoGen, this library provides the ability to set specific parameters as well as track API costs.\n", + "\n", + "You will need a Groq account and create an API key. [See their website for further details](https://groq.com/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Groq provides a number of models to use, included below. See the list of [models here (requires login)](https://console.groq.com/docs/models).\n", + "\n", + "See the sample `OAI_CONFIG_LIST` below showing how the Groq client class is used by specifying the `api_type` as `groq`.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"gpt-35-turbo\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"gpt-4-vision-preview\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"dalle\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"llama3-8b-8192\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " },\n", + " {\n", + " \"model\": \"llama3-70b-8192\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " },\n", + " {\n", + " \"model\": \"Mixtral 8x7b\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " },\n", + " {\n", + " \"model\": \"gemma-7b-it\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an alternative to the `api_key` key and value in the config, you can set the environment variable `GROQ_API_KEY` to your Groq key.\n", + "\n", + "Linux/Mac:\n", + "``` bash\n", + "export GROQ_API_KEY=\"your_groq_api_key_here\"\n", + "```\n", + "\n", + "Windows:\n", + "``` bash\n", + "set GROQ_API_KEY=your_groq_api_key_here\n", + "```\n", + "\n", + "## API parameters\n", + "\n", + "The following parameters can be added to your config for the Groq API. See [this link](https://console.groq.com/docs/text-chat) for further information on them.\n", + "\n", + "- frequency_penalty (number 0..1)\n", + "- max_tokens (integer >= 0)\n", + "- presence_penalty (number -2..2)\n", + "- seed (integer)\n", + "- temperature (number 0..2)\n", + "- top_p (number)\n", + "\n", + "Example:\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3-8b-8192\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\",\n", + " \"frequency_penalty\": 0.5,\n", + " \"max_tokens\": 2048,\n", + " \"presence_penalty\": 0.2,\n", + " \"seed\": 42,\n", + " \"temperature\": 0.5,\n", + " \"top_p\": 0.2\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Two-Agent Coding Example\n", + "\n", + "In this example, we run a two-agent chat with an AssistantAgent (primarily a coding agent) to generate code to count the number of prime numbers between 1 and 10,000 and then it will be executed.\n", + "\n", + "We'll use Meta's Llama 3 model which is suitable for coding." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "config_list = [\n", + " {\n", + " # Let's choose the Llama 3 model\n", + " \"model\": \"llama3-8b-8192\",\n", + " # Put your Groq API key here or put it into the GROQ_API_KEY environment variable.\n", + " \"api_key\": os.environ.get(\"GROQ_API_KEY\"),\n", + " # We specify the API Type as 'groq' so it uses the Groq client class\n", + " \"api_type\": \"groq\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, we have tweaked the system message so that the model doesn't return the termination keyword, which we've changed to FINISH, with the code block." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.coding import LocalCommandLineCodeExecutor\n", + "\n", + "# Setting up the code executor\n", + "workdir = Path(\"coding\")\n", + "workdir.mkdir(exist_ok=True)\n", + "code_executor = LocalCommandLineCodeExecutor(work_dir=workdir)\n", + "\n", + "# Setting up the agents\n", + "\n", + "# The UserProxyAgent will execute the code that the AssistantAgent provides\n", + "user_proxy_agent = UserProxyAgent(\n", + " name=\"User\",\n", + " code_execution_config={\"executor\": code_executor},\n", + " is_termination_msg=lambda msg: \"FINISH\" in msg.get(\"content\"),\n", + ")\n", + "\n", + "system_message = \"\"\"You are a helpful AI assistant who writes code and the user executes it.\n", + "Solve tasks using your coding and language skills.\n", + "In the following cases, suggest python code (in a python coding block) for the user to execute.\n", + "Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.\n", + "When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.\n", + "Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.\n", + "If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.\n", + "When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.\n", + "IMPORTANT: Wait for the user to execute your code and then you can reply with the word \"FINISH\". DO NOT OUTPUT \"FINISH\" after your code block.\"\"\"\n", + "\n", + "# The AssistantAgent, using Groq's model, will take the coding request and return code\n", + "assistant_agent = AssistantAgent(\n", + " name=\"Groq Assistant\",\n", + " system_message=system_message,\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to Groq Assistant):\n", + "\n", + "Provide code to count the number of prime numbers from 1 to 10000.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mGroq Assistant\u001b[0m (to User):\n", + "\n", + "Here's the plan to count the number of prime numbers from 1 to 10000:\n", + "\n", + "First, we need to write a helper function to check if a number is prime. A prime number is a number that is divisible only by 1 and itself.\n", + "\n", + "Then, we can use a loop to iterate through all numbers from 1 to 10000, check if each number is prime using our helper function, and count the number of prime numbers found.\n", + "\n", + "Here's the Python code to implement this plan:\n", + "```python\n", + "def is_prime(n):\n", + " if n <= 1:\n", + " return False\n", + " for i in range(2, int(n ** 0.5) + 1):\n", + " if n % i == 0:\n", + " return False\n", + " return True\n", + "\n", + "count = 0\n", + "for i in range(2, 10001):\n", + " if is_prime(i):\n", + " count += 1\n", + "\n", + "print(count)\n", + "```\n", + "Please execute this code, and I'll wait for the result.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Groq Assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: 1229\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mGroq Assistant\u001b[0m (to User):\n", + "\n", + "FINISH\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "# Start the chat, with the UserProxyAgent asking the AssistantAgent the message\n", + "chat_result = user_proxy_agent.initiate_chat(\n", + " assistant_agent,\n", + " message=\"Provide code to count the number of prime numbers from 1 to 10000.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Call Example\n", + "\n", + "In this example, instead of writing code, we will show how we can use Meta's Llama 3 model to perform parallel tool calling, where it recommends calling more than one tool at a time, using Groq's cloud inference.\n", + "\n", + "We'll use a simple travel agent assistant program where we have a couple of tools for weather and currency conversion.\n", + "\n", + "We start by importing libraries and setting up our configuration to use Meta's Llama 3 model and the `groq` client class." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import Literal\n", + "\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "\n", + "config_list = [\n", + " {\"api_type\": \"groq\", \"model\": \"llama3-8b-8192\", \"api_key\": os.getenv(\"GROQ_API_KEY\"), \"cache_seed\": None}\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create our two agents." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the agent for tool calling\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"\"\"For currency exchange and weather forecasting tasks,\n", + " only use the functions you have been provided with.\n", + " Output 'HAVE FUN!' when an answer has been provided.\"\"\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# Note that we have changed the termination string to be \"HAVE FUN!\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"HAVE FUN!\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the two functions, annotating them so that those descriptions can be passed through to the LLM.\n", + "\n", + "We associate them with the agents using `register_for_execution` for the user_proxy so it can execute the function and `register_for_llm` for the chatbot (powered by the LLM) so it can pass the function definitions to the LLM." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Currency Exchange function\n", + "\n", + "CurrencySymbol = Literal[\"USD\", \"EUR\"]\n", + "\n", + "# Define our function that we expect to call\n", + "\n", + "\n", + "def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float:\n", + " if base_currency == quote_currency:\n", + " return 1.0\n", + " elif base_currency == \"USD\" and quote_currency == \"EUR\":\n", + " return 1 / 1.1\n", + " elif base_currency == \"EUR\" and quote_currency == \"USD\":\n", + " return 1.1\n", + " else:\n", + " raise ValueError(f\"Unknown currencies {base_currency}, {quote_currency}\")\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Currency exchange calculator.\")\n", + "def currency_calculator(\n", + " base_amount: Annotated[float, \"Amount of currency in base_currency\"],\n", + " base_currency: Annotated[CurrencySymbol, \"Base currency\"] = \"USD\",\n", + " quote_currency: Annotated[CurrencySymbol, \"Quote currency\"] = \"EUR\",\n", + ") -> str:\n", + " quote_amount = exchange_rate(base_currency, quote_currency) * base_amount\n", + " return f\"{format(quote_amount, '.2f')} {quote_currency}\"\n", + "\n", + "\n", + "# Weather function\n", + "\n", + "\n", + "# Example function to make available to model\n", + "def get_current_weather(location, unit=\"fahrenheit\"):\n", + " \"\"\"Get the weather for some location\"\"\"\n", + " if \"chicago\" in location.lower():\n", + " return json.dumps({\"location\": \"Chicago\", \"temperature\": \"13\", \"unit\": unit})\n", + " elif \"san francisco\" in location.lower():\n", + " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"55\", \"unit\": unit})\n", + " elif \"new york\" in location.lower():\n", + " return json.dumps({\"location\": \"New York\", \"temperature\": \"11\", \"unit\": unit})\n", + " else:\n", + " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Weather forecast for US cities.\")\n", + "def weather_forecast(\n", + " location: Annotated[str, \"City name\"],\n", + ") -> str:\n", + " weather_details = get_current_weather(location=location)\n", + " weather = json.loads(weather_details)\n", + " return f\"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We pass through our customers message and run the chat.\n", + "\n", + "Finally, we ask the LLM to summarise the chat and print that out." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_hg7g): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\":\"New York\"}\n", + "\u001b[32m*************************************************************\u001b[0m\n", + "\u001b[32m***** Suggested tool call (call_hrsf): currency_calculator *****\u001b[0m\n", + "Arguments: \n", + "{\"base_amount\":123.45,\"base_currency\":\"EUR\",\"quote_currency\":\"USD\"}\n", + "\u001b[32m****************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION weather_forecast...\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION currency_calculator...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_hg7g) *****\u001b[0m\n", + "New York will be 11 degrees fahrenheit\n", + "\u001b[32m**************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_hrsf) *****\u001b[0m\n", + "135.80 USD\n", + "\u001b[32m**************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_ahwk): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\":\"New York\"}\n", + "\u001b[32m*************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "LLM SUMMARY: Based on the conversation, it's predicted that New York will be 11 degrees Fahrenheit. You also found out that 123.45 EUR is equal to 135.80 USD. Here are a few holiday tips:\n", + "\n", + "* Pack warm clothing for your visit to New York, as the temperature is expected to be quite chilly.\n", + "* Consider exchanging your money at a local currency exchange or an ATM since the exchange rate might not be as favorable in tourist areas.\n", + "* Make sure to check the estimated expenses for your holiday and adjust your budget accordingly.\n", + "\n", + "I hope you have a great trip!\n" + ] + } + ], + "source": [ + "# start the conversation\n", + "res = user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\",\n", + " summary_method=\"reflection_with_llm\",\n", + ")\n", + "\n", + "print(f\"LLM SUMMARY: {res.summary['content']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using its fast inference, Groq required less than 2 seconds for the whole chat!\n", + "\n", + "Additionally, Llama 3 was able to call both tools and pass through the right parameters. The `user_proxy` then executed them and this was passed back for Llama 3 to summarise the whole conversation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 0655b4341b419048c012ae4f1125b0405a3c7aca Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Thu, 4 Jul 2024 01:09:22 -0400 Subject: [PATCH 25/57] bump version (#3073) --- autogen/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen/version.py b/autogen/version.py index 77fc1e2ea295..93824aa1f87c 100644 --- a/autogen/version.py +++ b/autogen/version.py @@ -1 +1 @@ -__version__ = "0.2.31" +__version__ = "0.2.32" From f2119943310f297e903e34a733511d3a079b6493 Mon Sep 17 00:00:00 2001 From: William W Wang <107702013+wmwxwa@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:16:29 -0400 Subject: [PATCH 26/57] Update azure_cosmos_db.md (#3043) --- website/docs/ecosystem/azure_cosmos_db.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/ecosystem/azure_cosmos_db.md b/website/docs/ecosystem/azure_cosmos_db.md index 1ec29e40d771..0d1313bc14b7 100644 --- a/website/docs/ecosystem/azure_cosmos_db.md +++ b/website/docs/ecosystem/azure_cosmos_db.md @@ -7,7 +7,7 @@ Azure Cosmos DB is a fully managed [NoSQL](https://learn.microsoft.com/en-us/azu Your can simplify your application development by using this single database service for all your AI agent memory system needs, from [geo-replicated distributed cache](https://medium.com/@marcodesanctis2/using-azure-cosmos-db-as-your-persistent-geo-replicated-distributed-cache-b381ad80f8a0) to tracing/logging to [vector database](https://learn.microsoft.com/en-us/azure/cosmos-db/vector-database). -Learn more about how Azure Cosmos DB enhances the performance of your [AI agents](https://learn.microsoft.com/en-us/azure/cosmos-db/ai-agents). +Learn more about how Azure Cosmos DB enhances the performance of your [AI agent](https://learn.microsoft.com/en-us/azure/cosmos-db/ai-agents). - [Try Azure Cosmos DB free](https://learn.microsoft.com/en-us/azure/cosmos-db/try-free) - [Use Azure Cosmos DB lifetime free tier](https://learn.microsoft.com/en-us/azure/cosmos-db/free-tier) From 97b8e119d9db5048cb0ba8c386682c3ec661c30b Mon Sep 17 00:00:00 2001 From: Media <12145726+rihp@users.noreply.github.com> Date: Thu, 4 Jul 2024 20:26:47 +0200 Subject: [PATCH 27/57] Update AutoTX Link on Gallery.json (#3082) --- website/src/data/gallery.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/data/gallery.json b/website/src/data/gallery.json index 5ad6932fd92e..cf68764e241c 100644 --- a/website/src/data/gallery.json +++ b/website/src/data/gallery.json @@ -1,7 +1,7 @@ [ { "title": "AutoTx - Crypto Transactions Agent", - "link": "https://blog.polywrap.io/p/autotx-your-ai-powered-transaction", + "link": "https://www.agentcoin.org/blog/autotx", "description": "Generates on-chain transactions, which are submitted to a smart account so users can easily approve & execute them.", "image": "autotx.png", "tags": ["tools", "groupchat", "app", "blockchain"] From 0a82b3eb76f4e16642ac4bf78e790c484e4c3ffc Mon Sep 17 00:00:00 2001 From: Shaun Date: Sat, 6 Jul 2024 00:59:58 +0800 Subject: [PATCH 28/57] fix: support openai service account apikey format (#3078) --- autogen/oai/openai_utils.py | 2 +- test/oai/test_utils.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 0c8a0a413375..749727d952c0 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -96,7 +96,7 @@ def is_valid_api_key(api_key: str) -> bool: Returns: bool: A boolean that indicates if input is valid OpenAI API key. """ - api_key_re = re.compile(r"^sk-(proj-)?[A-Za-z0-9]{32,}$") + api_key_re = re.compile(r"^sk-([A-Za-z0-9]+(-+[A-Za-z0-9]+)*-)?[A-Za-z0-9]{32,}$") return bool(re.fullmatch(api_key_re, api_key)) diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py index 99f8d8d24e8b..96956d07d90d 100755 --- a/test/oai/test_utils.py +++ b/test/oai/test_utils.py @@ -424,6 +424,12 @@ def test_is_valid_api_key(): assert is_valid_api_key("sk-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS") assert is_valid_api_key("sk-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS1212121221212sssXX") assert is_valid_api_key("sk-proj-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-0-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-aut0gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-aut0-gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-aut0--gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert not is_valid_api_key("sk-aut0-gen--asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert not is_valid_api_key("sk--aut0-gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") assert is_valid_api_key(MOCK_OPEN_AI_API_KEY) From 82903f5f89a4f514b267911e56c3032b339f2581 Mon Sep 17 00:00:00 2001 From: Prithvi Date: Fri, 5 Jul 2024 16:10:58 -0700 Subject: [PATCH 29/57] [.Net] Update FunctionCallTemplate.tt to encode `"` (#3068) * Update FunctionCallTemplate.tt changed the desscription assigning to handle double quotes in comments and prevent the generated code from breaking. * Added the necessary changes Fixed handling of double quotes in descriptions within FunctionCallTemplate.tt. Standardized newline characters to ensure consistency. Updated test cases in FunctionCallTemplateEncodingTests to verify correct encoding of double quotes in descriptions. Cleaned up unnecessary using directives in FunctionCallTemplateEncodingTests. Aligned the template with the approved test output. * test cases passing Test cases passing like `Starting test execution, please wait... A total of 1 test files matched the specified pattern. Passed! - Failed: 0, Passed: 9, Skipped: 0, Total: 9, Duration: 66 ms - AutoGen.SourceGenerator.Tests.dll (net8.0)` * Delete FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt Deleted the ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt successfully! * Revert "Delete FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt" This reverts commit 7a6ea9cf0d5831ba7a5c445a7f9b64da713c600d. --------- Co-authored-by: Xiaoyun Zhang --- .../Template/FunctionCallTemplate.cs | 479 +++++++----------- .../Template/FunctionCallTemplate.tt | 4 +- ...ests.TestFunctionCallTemplate.approved.txt | 2 +- .../FunctionCallTemplateEncodingTests.cs | 92 ++++ 4 files changed, 271 insertions(+), 306 deletions(-) create mode 100644 dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs diff --git a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs index 40adbdcde47c..cb3574a8b1ea 100644 --- a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs +++ b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs @@ -1,31 +1,30 @@ -๏ปฟ// ------------------------------------------------------------------------------ +๏ปฟ//------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version: 17.0.0.0 -// +// // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. // -// ------------------------------------------------------------------------------ -namespace AutoGen.SourceGenerator.Template -{ +//------------------------------------------------------------------------------ + +namespace AutoGen.SourceGenerator.Template { using System.Linq; using System.Collections.Generic; using Microsoft.CodeAnalysis; using System; - /// - /// Class to produce the template output - /// - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] - internal partial class FunctionCallTemplate : FunctionCallTemplateBase - { - /// - /// Create the template output - /// - public virtual string TransformText() - { - this.Write("๏ปฟ"); + + internal partial class FunctionCallTemplate : FunctionCallTemplateBase { + + +public string NameSpace {get; set;} +public string ClassName {get; set;} +public IEnumerable FunctionContracts {get; set;} +public bool IsStatic {get; set;} = false; + + + public virtual string TransformText() { + this.GenerationEnvironment = null; this.Write(@"//---------------------- // // This code was generated by a tool. @@ -42,407 +41,281 @@ public virtual string TransformText() if (!String.IsNullOrEmpty(NameSpace)) { this.Write("namespace "); this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); - this.Write("\r\n{\r\n"); + this.Write("\n{\n"); } this.Write(" public partial class "); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); - this.Write("\r\n {\r\n"); + this.Write("\n {\n"); foreach (var functionContract in FunctionContracts) { - this.Write("\r\n private class "); + this.Write("\n private class "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionSchemaClassName())); - this.Write("\r\n {\r\n"); + this.Write("\n {\n"); foreach (var parameter in functionContract.Parameters) { if (parameter.IsOptional) { this.Write(" [JsonPropertyName(@\""); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write("\")]\r\n\t\t\tpublic "); + this.Write("\")]\n\t\t\tpublic "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Type)); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); this.Write(" {get; set;} = "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.DefaultValue)); - this.Write(";\r\n"); + this.Write(";\n"); } else { this.Write(" [JsonPropertyName(@\""); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write("\")]\r\n\t\t\tpublic "); + this.Write("\")]\n\t\t\tpublic "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Type)); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write(" {get; set;}\r\n"); + this.Write(" {get; set;}\n"); } } - this.Write(" }\r\n\r\n public "); + this.Write(" }\n\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ReturnType)); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionWrapperName())); - this.Write("(string arguments)\r\n {\r\n var schema = JsonSerializer.Deserializ" + - "e<"); + this.Write("(string arguments)\n {\n var schema = JsonSerializer.Deserialize<" + + ""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionSchemaClassName())); - this.Write(">(\r\n arguments, \r\n new JsonSerializerOptions\r\n " + - " {\r\n PropertyNamingPolicy = JsonNamingPolicy.CamelC" + - "ase,\r\n });\r\n"); + this.Write(">(\n arguments, \n new JsonSerializerOptions\n " + + " {\n PropertyNamingPolicy = JsonNamingPolicy.CamelCase," + + "\n });\n"); var argumentLists = string.Join(", ", functionContract.Parameters.Select(p => $"schema.{p.Name}")); - this.Write("\r\n return "); + this.Write("\n return "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Name)); this.Write("("); this.Write(this.ToStringHelper.ToStringWithCulture(argumentLists)); - this.Write(");\r\n }\r\n\r\n public FunctionContract "); + this.Write(");\n }\n\n public FunctionContract "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionContractName())); - this.Write("\r\n {\r\n get => new FunctionContract\r\n {\r\n"); + this.Write("\n {\n get => new FunctionContract\n {\n"); if (functionContract.Namespace != null) { this.Write(" Namespace = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Namespace)); - this.Write("\",\r\n"); + this.Write("\",\n"); } if (functionContract.ClassName != null) { this.Write(" ClassName = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ClassName)); - this.Write("\",\r\n"); + this.Write("\",\n"); } if (functionContract.Name != null) { this.Write(" Name = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Name)); - this.Write("\",\r\n"); + this.Write("\",\n"); } if (functionContract.Description != null) { this.Write(" Description = @\""); - this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Description)); - this.Write("\",\r\n"); + this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Description.Replace("\"", "\"\""))); + this.Write("\",\n"); } if (functionContract.ReturnType != null) { this.Write(" ReturnType = typeof("); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ReturnType)); - this.Write("),\r\n"); + this.Write("),\n"); } if (functionContract.ReturnDescription != null) { this.Write(" ReturnDescription = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ReturnDescription)); - this.Write("\",\r\n"); + this.Write("\",\n"); } if (functionContract.Parameters != null) { this.Write(" Parameters = new global::AutoGen.Core.FunctionParameterContract[]" + - "\r\n {\r\n"); + "\n {\n"); foreach (var parameter in functionContract.Parameters) { - this.Write(" new FunctionParameterContract\r\n {\r\n"); + this.Write(" new FunctionParameterContract\n {\n"); if (parameter.Name != null) { this.Write(" Name = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write("\",\r\n"); + this.Write("\",\n"); } if (parameter.Description != null) { this.Write(" Description = @\""); - this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Description)); - this.Write("\",\r\n"); + this.Write(this.ToStringHelper.ToStringWithCulture( parameter.Description.Replace("\"", "\"\"") )); + this.Write("\",\n"); } if (parameter.Type != null) { this.Write(" ParameterType = typeof("); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Type)); - this.Write("),\r\n"); + this.Write("),\n"); } this.Write(" IsRequired = "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.IsOptional ? "false" : "true")); - this.Write(",\r\n"); + this.Write(",\n"); if (parameter.DefaultValue != null) { this.Write(" DefaultValue = "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.DefaultValue)); - this.Write(",\r\n"); + this.Write(",\n"); } - this.Write(" },\r\n"); + this.Write(" },\n"); } - this.Write(" },\r\n"); + this.Write(" },\n"); } - this.Write(" };\r\n }\r\n\r\n public global::Azure.AI.OpenAI.FunctionDefin" + - "ition "); + this.Write(" };\n }\n\n public global::Azure.AI.OpenAI.FunctionDefiniti" + + "on "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionDefinitionName())); - this.Write("\r\n {\r\n get => this."); + this.Write("\n {\n get => this."); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionContractName())); - this.Write(".ToOpenAIFunctionDefinition();\r\n }\r\n"); + this.Write(".ToOpenAIFunctionDefinition();\n }\n"); } - this.Write(" }\r\n"); + this.Write(" }\n"); if (!String.IsNullOrEmpty(NameSpace)) { - this.Write("}\r\n"); + this.Write("}\n"); } - this.Write("\r\n"); + this.Write("\n"); return this.GenerationEnvironment.ToString(); } - -public string NameSpace {get; set;} -public string ClassName {get; set;} -public IEnumerable FunctionContracts {get; set;} -public bool IsStatic {get; set;} = false; - + + public virtual void Initialize() { + } } - #region Base class - /// - /// Base class for this transformation - /// - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] - internal class FunctionCallTemplateBase - { - #region Fields - private global::System.Text.StringBuilder generationEnvironmentField; - private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; - private global::System.Collections.Generic.List indentLengthsField; - private string currentIndentField = ""; - private bool endsWithNewline; - private global::System.Collections.Generic.IDictionary sessionField; - #endregion - #region Properties - /// - /// The string builder that generation-time code is using to assemble generated output - /// - public System.Text.StringBuilder GenerationEnvironment - { - get - { - if ((this.generationEnvironmentField == null)) - { - this.generationEnvironmentField = new global::System.Text.StringBuilder(); - } - return this.generationEnvironmentField; + + public class FunctionCallTemplateBase { + + private global::System.Text.StringBuilder builder; + + private global::System.Collections.Generic.IDictionary session; + + private global::System.CodeDom.Compiler.CompilerErrorCollection errors; + + private string currentIndent = string.Empty; + + private global::System.Collections.Generic.Stack indents; + + private ToStringInstanceHelper _toStringHelper = new ToStringInstanceHelper(); + + public virtual global::System.Collections.Generic.IDictionary Session { + get { + return this.session; } - set - { - this.generationEnvironmentField = value; + set { + this.session = value; } } - /// - /// The error collection for the generation process - /// - public System.CodeDom.Compiler.CompilerErrorCollection Errors - { - get - { - if ((this.errorsField == null)) - { - this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + + public global::System.Text.StringBuilder GenerationEnvironment { + get { + if ((this.builder == null)) { + this.builder = new global::System.Text.StringBuilder(); } - return this.errorsField; + return this.builder; + } + set { + this.builder = value; } } - /// - /// A list of the lengths of each indent that was added with PushIndent - /// - private System.Collections.Generic.List indentLengths - { - get - { - if ((this.indentLengthsField == null)) - { - this.indentLengthsField = new global::System.Collections.Generic.List(); + + protected global::System.CodeDom.Compiler.CompilerErrorCollection Errors { + get { + if ((this.errors == null)) { + this.errors = new global::System.CodeDom.Compiler.CompilerErrorCollection(); } - return this.indentLengthsField; + return this.errors; } } - /// - /// Gets the current indent we use when adding lines to the output - /// - public string CurrentIndent - { - get - { - return this.currentIndentField; + + public string CurrentIndent { + get { + return this.currentIndent; } } - /// - /// Current transformation session - /// - public virtual global::System.Collections.Generic.IDictionary Session - { - get - { - return this.sessionField; - } - set - { - this.sessionField = value; + + private global::System.Collections.Generic.Stack Indents { + get { + if ((this.indents == null)) { + this.indents = new global::System.Collections.Generic.Stack(); + } + return this.indents; } } - #endregion - #region Transform-time helpers - /// - /// Write text directly into the generated output - /// - public void Write(string textToAppend) - { - if (string.IsNullOrEmpty(textToAppend)) - { - return; - } - // If we're starting off, or if the previous text ended with a newline, - // we have to append the current indent first. - if (((this.GenerationEnvironment.Length == 0) - || this.endsWithNewline)) - { - this.GenerationEnvironment.Append(this.currentIndentField); - this.endsWithNewline = false; - } - // Check if the current text ends with a newline - if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) - { - this.endsWithNewline = true; - } - // This is an optimization. If the current indent is "", then we don't have to do any - // of the more complex stuff further down. - if ((this.currentIndentField.Length == 0)) - { - this.GenerationEnvironment.Append(textToAppend); - return; - } - // Everywhere there is a newline in the text, add an indent after it - textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); - // If the text ends with a newline, then we should strip off the indent added at the very end - // because the appropriate indent will be added when the next time Write() is called - if (this.endsWithNewline) - { - this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); - } - else - { - this.GenerationEnvironment.Append(textToAppend); + + public ToStringInstanceHelper ToStringHelper { + get { + return this._toStringHelper; } } - /// - /// Write text directly into the generated output - /// - public void WriteLine(string textToAppend) - { - this.Write(textToAppend); - this.GenerationEnvironment.AppendLine(); - this.endsWithNewline = true; + + public void Error(string message) { + this.Errors.Add(new global::System.CodeDom.Compiler.CompilerError(null, -1, -1, null, message)); } - /// - /// Write formatted text directly into the generated output - /// - public void Write(string format, params object[] args) - { - this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + + public void Warning(string message) { + global::System.CodeDom.Compiler.CompilerError val = new global::System.CodeDom.Compiler.CompilerError(null, -1, -1, null, message); + val.IsWarning = true; + this.Errors.Add(val); } - /// - /// Write formatted text directly into the generated output - /// - public void WriteLine(string format, params object[] args) - { - this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); + + public string PopIndent() { + if ((this.Indents.Count == 0)) { + return string.Empty; + } + int lastPos = (this.currentIndent.Length - this.Indents.Pop()); + string last = this.currentIndent.Substring(lastPos); + this.currentIndent = this.currentIndent.Substring(0, lastPos); + return last; } - /// - /// Raise an error - /// - public void Error(string message) - { - System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); - error.ErrorText = message; - this.Errors.Add(error); + + public void PushIndent(string indent) { + this.Indents.Push(indent.Length); + this.currentIndent = (this.currentIndent + indent); } - /// - /// Raise a warning - /// - public void Warning(string message) - { - System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); - error.ErrorText = message; - error.IsWarning = true; - this.Errors.Add(error); + + public void ClearIndent() { + this.currentIndent = string.Empty; + this.Indents.Clear(); } - /// - /// Increase the indent - /// - public void PushIndent(string indent) - { - if ((indent == null)) - { - throw new global::System.ArgumentNullException("indent"); - } - this.currentIndentField = (this.currentIndentField + indent); - this.indentLengths.Add(indent.Length); + + public void Write(string textToAppend) { + this.GenerationEnvironment.Append(textToAppend); } - /// - /// Remove the last indent that was added with PushIndent - /// - public string PopIndent() - { - string returnValue = ""; - if ((this.indentLengths.Count > 0)) - { - int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; - this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); - if ((indentLength > 0)) - { - returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); - this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); - } - } - return returnValue; + + public void Write(string format, params object[] args) { + this.GenerationEnvironment.AppendFormat(format, args); } - /// - /// Remove any indentation - /// - public void ClearIndent() - { - this.indentLengths.Clear(); - this.currentIndentField = ""; + + public void WriteLine(string textToAppend) { + this.GenerationEnvironment.Append(this.currentIndent); + this.GenerationEnvironment.AppendLine(textToAppend); } - #endregion - #region ToString Helpers - /// - /// Utility class to produce culture-oriented representation of an object as a string. - /// - public class ToStringInstanceHelper - { - private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; - /// - /// Gets or sets format provider to be used by ToStringWithCulture method. - /// - public System.IFormatProvider FormatProvider - { - get - { - return this.formatProviderField ; + + public void WriteLine(string format, params object[] args) { + this.GenerationEnvironment.Append(this.currentIndent); + this.GenerationEnvironment.AppendFormat(format, args); + this.GenerationEnvironment.AppendLine(); + } + + public class ToStringInstanceHelper { + + private global::System.IFormatProvider formatProvider = global::System.Globalization.CultureInfo.InvariantCulture; + + public global::System.IFormatProvider FormatProvider { + get { + return this.formatProvider; } - set - { - if ((value != null)) - { - this.formatProviderField = value; + set { + if ((value != null)) { + this.formatProvider = value; } } } - /// - /// This is called from the compile/run appdomain to convert objects within an expression block to a string - /// - public string ToStringWithCulture(object objectToConvert) - { - if ((objectToConvert == null)) - { + + public string ToStringWithCulture(object objectToConvert) { + if ((objectToConvert == null)) { throw new global::System.ArgumentNullException("objectToConvert"); } - System.Type t = objectToConvert.GetType(); - System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { - typeof(System.IFormatProvider)}); - if ((method == null)) - { - return objectToConvert.ToString(); + global::System.Type type = objectToConvert.GetType(); + global::System.Type iConvertibleType = typeof(global::System.IConvertible); + if (iConvertibleType.IsAssignableFrom(type)) { + return ((global::System.IConvertible)(objectToConvert)).ToString(this.formatProvider); } - else - { - return ((string)(method.Invoke(objectToConvert, new object[] { - this.formatProviderField }))); + global::System.Reflection.MethodInfo methInfo = type.GetMethod("ToString", new global::System.Type[] { + iConvertibleType}); + if ((methInfo != null)) { + return ((string)(methInfo.Invoke(objectToConvert, new object[] { + this.formatProvider}))); } + return objectToConvert.ToString(); } } - private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); - /// - /// Helper to produce culture-oriented representation of an object as a string - /// - public ToStringInstanceHelper ToStringHelper - { - get - { - return this.toStringHelperField; - } - } - #endregion } - #endregion } diff --git a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt index 0d1b221c35c8..c7a13ce1fe92 100644 --- a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt +++ b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt @@ -63,7 +63,7 @@ namespace <#=NameSpace#> Name = @"<#=functionContract.Name#>", <#}#> <#if (functionContract.Description != null) {#> - Description = @"<#=functionContract.Description#>", + Description = @"<#=functionContract.Description.Replace("\"", "\"\"")#>", <#}#> <#if (functionContract.ReturnType != null) {#> ReturnType = typeof(<#=functionContract.ReturnType#>), @@ -81,7 +81,7 @@ namespace <#=NameSpace#> Name = @"<#=parameter.Name#>", <#}#> <#if (parameter.Description != null) {#> - Description = @"<#=parameter.Description#>", + Description = @"<#= parameter.Description.Replace("\"", "\"\"") #>", <#}#> <#if (parameter.Type != null) {#> ParameterType = typeof(<#=parameter.Type#>), diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt b/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt index 0439febc52c7..a5ffa3c8248d 100644 --- a/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt @@ -1,4 +1,4 @@ -๏ปฟ๏ปฟ//---------------------- +๏ปฟ//---------------------- // // This code was generated by a tool. // diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs new file mode 100644 index 000000000000..8ca6e31a8a40 --- /dev/null +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs @@ -0,0 +1,92 @@ +// Using directives +using System.Text.Json; // Needed for JsonSerializer +using Xunit; // Needed for Fact and Assert +using AutoGen.SourceGenerator.Template; // Needed for FunctionCallTemplate + +namespace AutoGen.SourceGenerator.Tests +{ + public class FunctionCallTemplateEncodingTests + { + private readonly JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + WriteIndented = true, + }; + + [Fact] + public void FunctionDescription_Should_Encode_DoubleQuotes() + { + // Arrange + var functionContracts = new List + { + new SourceGeneratorFunctionContract + { + Name = "TestFunction", + Description = "This is a \"test\" function", + Parameters = new SourceGeneratorParameterContract[] + { + new SourceGeneratorParameterContract + { + Name = "param1", + Description = "This is a \"parameter\" description", + Type = "string", + IsOptional = false + } + }, + ReturnType = "void" + } + }; + + var template = new FunctionCallTemplate + { + NameSpace = "TestNamespace", + ClassName = "TestClass", + FunctionContracts = functionContracts + }; + + // Act + var result = template.TransformText(); + + // Assert + Assert.Contains("Description = @\"This is a \"\"test\"\" function\"", result); + Assert.Contains("Description = @\"This is a \"\"parameter\"\" description\"", result); + } + + [Fact] + public void ParameterDescription_Should_Encode_DoubleQuotes() + { + // Arrange + var functionContracts = new List + { + new SourceGeneratorFunctionContract + { + Name = "TestFunction", + Description = "This is a test function", + Parameters = new SourceGeneratorParameterContract[] + { + new SourceGeneratorParameterContract + { + Name = "param1", + Description = "This is a \"parameter\" description", + Type = "string", + IsOptional = false + } + }, + ReturnType = "void" + } + }; + + var template = new FunctionCallTemplate + { + NameSpace = "TestNamespace", + ClassName = "TestClass", + FunctionContracts = functionContracts + }; + + // Act + var result = template.TransformText(); + + // Assert + Assert.Contains("Description = @\"This is a \"\"parameter\"\" description\"", result); + } + } +} \ No newline at end of file From f55a98f32bf791b171519f0f8f7bf2bf747c3101 Mon Sep 17 00:00:00 2001 From: Zoltan Lux Date: Mon, 8 Jul 2024 00:11:35 +0200 Subject: [PATCH 30/57] Demo Notebook for Using Gemini with VertexAI (#3032) * add notebook for using Gemini with VertexAI * add missing image * remove part with workload identity federation * Spelling * Capitalisation and tweak on config note. * autogen gemini gcp image * fix formatting * move gemini vertexai notebook to website/docs/topics/non-openai-models * Adjust license Co-authored-by: Chi Wang * remove auto-generated cell --------- Co-authored-by: Mark Sze <66362098+marklysze@users.noreply.github.com> Co-authored-by: Chi Wang --- .../cloud-gemini_vertexai.ipynb | 796 ++++++++++++++++++ website/static/img/create_gcp_svc.png | 3 + 2 files changed, 799 insertions(+) create mode 100644 website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb create mode 100644 website/static/img/create_gcp_svc.png diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb new file mode 100644 index 000000000000..eaec2b72b268 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -0,0 +1,796 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Use AutoGen with Gemini via VertexAI\n", + "\n", + "This notebook demonstrates how to use Autogen with Gemini via Vertex AI, which enables enhanced authentication method that also supports enterprise requirements using service accounts or even a personal Google cloud account.\n", + "\n", + "## Requirements\n", + "\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please install with the [gemini] option:\n", + "```bash\n", + "pip install \"pyautogen[gemini]\"\n", + "```\n", + "\n", + "### Google Cloud Account\n", + "To use VertexAI a Google Cloud account is needed. If you do not have one yet, just sign up for a free trial [here](https://cloud.google.com).\n", + "\n", + "Login to your account at [console.cloud.google.com](https://console.cloud.google.com)\n", + "\n", + "In the next step we create a Google Cloud project, which is needed for VertexAI. The official guide for creating a project is available is [here](https://developers.google.com/workspace/guides/create-project). \n", + "\n", + "We will name our project Autogen-with-Gemini.\n", + "\n", + "### Enable Google Cloud APIs\n", + "\n", + "If you wish to use Gemini with your personal account, then creating a Google Cloud account is enough. However, if a service account is needed, then a few extra steps are needed.\n", + "\n", + "#### Enable API for Gemini\n", + " * For enabling Gemini for Google Cloud search for \"api\" and select Enabled APIs & services. \n", + " * Then click ENABLE APIS AND SERVICES. \n", + " * Search for Gemini, and select Gemini for Google Cloud.
A direct link will look like this for our autogen-with-gemini project:\n", + "https://console.cloud.google.com/apis/library/cloudaicompanion.googleapis.com?project=autogen-with-gemini&supportedpurview=project\n", + "* Click ENABLE for Gemini for Google Cloud.\n", + "\n", + "### Enable API for Vertex AI\n", + "* For enabling Vertex AI for Google Cloud search for \"api\" and select Enabled APIs & services. \n", + "* Then click ENABLE APIS AND SERVICES. \n", + "* Search for Vertex AI, and select Vertex AI API.
A direct link for our autogen-with-gemini will be: https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=autogen-with-gemini\n", + "* Click ENABLE Vertex AI API for Google Cloud.\n", + "\n", + "### Create a Service Account\n", + "\n", + "You can find an overview of service accounts [can be found in the cloud console](https://console.cloud.google.com/iam-admin/serviceaccounts)\n", + "\n", + "Detailed guide: https://cloud.google.com/iam/docs/service-accounts-create\n", + "\n", + "A service account can be created within the scope of a project, so a project needs to be selected.\n", + "\n", + "
\n", + "\n", + "
\n", + "\n", + "For the sake of simplicity we will assign the Editor role to our service account for autogen on our Autogen-with-Gemini Google Cloud project.\n", + "\n", + "* Under IAM & Admin > Service Account select the newly created service accounts, and click the option \"Manage keys\" among the items. \n", + "* From the \"ADD KEY\" dropdown select \"Create new key\" and select the JSON format and click CREATE.\n", + " * The new key will be downloaded automatically. \n", + "* You can then upload the service account key file to the from where you will be running autogen. \n", + " * Please consider restricting the permissions on the key file. For example, you could run `chmod 600 autogen-with-gemini-service-account-key.json` if your keyfile is called autogen-with-gemini-service-account-key.json." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2023-02-13T23:40:52.317406Z", + "iopub.status.busy": "2023-02-13T23:40:52.316561Z", + "iopub.status.idle": "2023-02-13T23:40:52.321193Z", + "shell.execute_reply": "2023-02-13T23:40:52.320628Z" + } + }, + "outputs": [], + "source": [ + "# %pip install \"pyautogen[gemini]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2023-02-13T23:40:54.634335Z", + "iopub.status.busy": "2023-02-13T23:40:54.633929Z", + "iopub.status.idle": "2023-02-13T23:40:56.105700Z", + "shell.execute_reply": "2023-02-13T23:40:56.105085Z" + }, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "import autogen" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Configure Authentication\n", + "\n", + "Authentication happens using standard [Google Cloud authentication methods](https://cloud.google.com/docs/authentication),
which means\n", + "that either an already active session can be reused, or by specifying the Google application credentials of a service account.\n", + "\n", + "#### Use Service Account Keyfile\n", + "\n", + "The Google Cloud service account can be specified by setting the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to the JSON key file of the service account.
\n", + "\n", + "We could even just directly set the environment variable, or we can add the `\"google_application_credentials\"` key with the respective value for our model in the OAI_CONFIG_LIST.\n", + "\n", + "#### Use the Google Default Credentials\n", + "\n", + "If you are using [Cloud Shell](https://shell.cloud.google.com/cloudshell) or [Cloud Shell editor](https://shell.cloud.google.com/cloudshell/editor) in Google Cloud,
then you are already authenticated. If you have the Google Cloud SDK installed locally,
then you can login by running `gcloud auth login` in the command line. \n", + "\n", + "Detailed instructions for installing the Google Cloud SDK can be found [here](https://cloud.google.com/sdk/docs/install)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example Config List\n", + "The config could look like the following (change `project_id` and `google_application_credentials`):\n", + "```python\n", + "config_list = [\n", + " {\n", + " \"model\": \"gemini-pro\",\n", + " \"api_type\": \"google\",\n", + " \"project_id\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\"\n", + " },\n", + " {\n", + " \"model\": \"gemini-1.5-pro-001\",\n", + " \"api_type\": \"google\",\n", + " \"project_id\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\"\n", + " },\n", + " {\n", + " \"model\": \"gemini-1.5-pro\",\n", + " \"api_type\": \"google\",\n", + " \"project\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\",\n", + " \"google_application_credentials\": \"autogen-with-gemini-service-account-key.json\"\n", + " },\n", + " {\n", + " \"model\": \"gemini-pro-vision\",\n", + " \"api_type\": \"google\",\n", + " \"project_id\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\"\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Configure Safety Settings for VertexAI\n", + "Configuring safety settings for VertexAI is slightly different, as we have to use the speicialized safety setting object types instead of plain strings" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from vertexai.generative_models import (\n", + " GenerationConfig,\n", + " GenerativeModel,\n", + " HarmBlockThreshold,\n", + " HarmCategory,\n", + " Part,\n", + ")\n", + "\n", + "safety_settings = {\n", + " HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + " HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + " HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + " HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union\n", + "\n", + "import chromadb\n", + "from PIL import Image\n", + "from termcolor import colored\n", + "\n", + "from autogen import Agent, AssistantAgent, ConversableAgent, UserProxyAgent\n", + "from autogen.agentchat.contrib.img_utils import _to_pil, get_image_data\n", + "from autogen.agentchat.contrib.multimodal_conversable_agent import MultimodalConversableAgent\n", + "from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent\n", + "from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent\n", + "from autogen.code_utils import DEFAULT_MODEL, UNKNOWN, content_str, execute_code, extract_code, infer_lang" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "config_list_gemini = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"model\": [\"gemini-1.5-pro\"],\n", + " },\n", + ")\n", + "\n", + "config_list_gemini_vision = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"model\": [\"gemini-pro-vision\"],\n", + " },\n", + ")\n", + "\n", + "for config_list in [config_list_gemini, config_list_gemini_vision]:\n", + " for config_list_item in config_list:\n", + " config_list_item[\"safety_settings\"] = safety_settings\n", + "\n", + "seed = 25 # for caching" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "\n", + " Compute the integral of the function f(x)=x^2 on the interval 0 to 1 using a Python script, \n", + " which returns the value of the definite integral.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + "Plan:\n", + "1. (Code) Use Python's numerical integration library to compute the integral.\n", + "2. (Language) Output the result.\n", + "\n", + "```python\n", + "# filename: integral.py\n", + "import scipy.integrate\n", + "\n", + "f = lambda x: x**2\n", + "result, error = scipy.integrate.quad(f, 0, 1)\n", + "\n", + "print(f\"The definite integral of x^2 from 0 to 1 is: {result}\")\n", + "```\n", + "\n", + "Let me know when you have executed the code. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "The definite integral of x^2 from 0 to 1 is: 0.33333333333333337\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + "The code executed successfully and returned the value of the definite integral as approximately 0.33333333333333337. \n", + "\n", + "This aligns with the analytical solution:\n", + "\n", + "The integral of x^2 is (x^3)/3. Evaluating this from 0 to 1 gives us (1^3)/3 - (0^3)/3 = 1/3 = 0.33333...\n", + "\n", + "Therefore, the answer is verified to be correct.\n", + "\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "assistant = AssistantAgent(\n", + " \"assistant\", llm_config={\"config_list\": config_list_gemini, \"seed\": seed}, max_consecutive_auto_reply=3\n", + ")\n", + "\n", + "user_proxy = UserProxyAgent(\n", + " \"user_proxy\",\n", + " code_execution_config={\"work_dir\": \"coding\", \"use_docker\": False},\n", + " human_input_mode=\"NEVER\",\n", + " is_termination_msg=lambda x: content_str(x.get(\"content\")).find(\"TERMINATE\") >= 0,\n", + ")\n", + "\n", + "result = user_proxy.initiate_chat(\n", + " assistant,\n", + " message=\"\"\"\n", + " Compute the integral of the function f(x)=x^2 on the interval 0 to 1 using a Python script,\n", + " which returns the value of the definite integral.\"\"\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example with Gemini Multimodal\n", + "Authentication is the same for vision models as for the text based Gemini models" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to Gemini Vision):\n", + "\n", + "Describe what is in this image?\n", + ".\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mGemini Vision\u001b[0m (to user_proxy):\n", + "\n", + " The image shows a taxonomy of different types of conversational agents. The taxonomy is based on two dimensions: agent customization and flexible conversation patterns. Agent customization refers to the ability of the agent to be tailored to the individual user. Flexible conversation patterns refer to the ability of the agent to engage in different types of conversations, such as joint chat and hierarchical chat.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "ChatResult(chat_id=None, chat_history=[{'content': 'Describe what is in this image?\\n.', 'role': 'assistant'}, {'content': ' The image shows a taxonomy of different types of conversational agents. The taxonomy is based on two dimensions: agent customization and flexible conversation patterns. Agent customization refers to the ability of the agent to be tailored to the individual user. Flexible conversation patterns refer to the ability of the agent to engage in different types of conversations, such as joint chat and hierarchical chat.', 'role': 'user'}], summary=' The image shows a taxonomy of different types of conversational agents. The taxonomy is based on two dimensions: agent customization and flexible conversation patterns. Agent customization refers to the ability of the agent to be tailored to the individual user. Flexible conversation patterns refer to the ability of the agent to engage in different types of conversations, such as joint chat and hierarchical chat.', cost={'usage_including_cached_inference': {'total_cost': 0.0002385, 'gemini-pro-vision': {'cost': 0.0002385, 'prompt_tokens': 267, 'completion_tokens': 70, 'total_tokens': 337}}, 'usage_excluding_cached_inference': {'total_cost': 0.0002385, 'gemini-pro-vision': {'cost': 0.0002385, 'prompt_tokens': 267, 'completion_tokens': 70, 'total_tokens': 337}}}, human_input=[])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "image_agent = MultimodalConversableAgent(\n", + " \"Gemini Vision\", llm_config={\"config_list\": config_list_gemini_vision, \"seed\": seed}, max_consecutive_auto_reply=1\n", + ")\n", + "\n", + "user_proxy = UserProxyAgent(\"user_proxy\", human_input_mode=\"NEVER\", max_consecutive_auto_reply=0)\n", + "\n", + "user_proxy.initiate_chat(\n", + " image_agent,\n", + " message=\"\"\"Describe what is in this image?\n", + ".\"\"\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "front_matter": { + "description": "Using Gemini with AutoGen via VertexAI", + "tags": [ + "gemini", + "vertexai" + ] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "vscode": { + "interpreter": { + "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1" + } + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2d910cfd2d2a4fc49fc30fbbdc5576a7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "454146d0f7224f038689031002906e6f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e4ae2b6f5a974fd4bafb6abb9d12ff26", + "IPY_MODEL_577e1e3cc4db4942b0883577b3b52755", + "IPY_MODEL_b40bdfb1ac1d4cffb7cefcb870c64d45" + ], + "layout": "IPY_MODEL_dc83c7bff2f241309537a8119dfc7555", + "tabbable": null, + "tooltip": null + } + }, + "577e1e3cc4db4942b0883577b3b52755": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2d910cfd2d2a4fc49fc30fbbdc5576a7", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_74a6ba0c3cbc4051be0a83e152fe1e62", + "tabbable": null, + "tooltip": null, + "value": 1 + } + }, + "6086462a12d54bafa59d3c4566f06cb2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "74a6ba0c3cbc4051be0a83e152fe1e62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7d3f3d9e15894d05a4d188ff4f466554": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b40bdfb1ac1d4cffb7cefcb870c64d45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f1355871cc6f4dd4b50d9df5af20e5c8", + "placeholder": "โ€‹", + "style": "IPY_MODEL_ca245376fd9f4354af6b2befe4af4466", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:00<00:00, 44.69it/s]" + } + }, + "ca245376fd9f4354af6b2befe4af4466": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "dc83c7bff2f241309537a8119dfc7555": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e4ae2b6f5a974fd4bafb6abb9d12ff26": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6086462a12d54bafa59d3c4566f06cb2", + "placeholder": "โ€‹", + "style": "IPY_MODEL_7d3f3d9e15894d05a4d188ff4f466554", + "tabbable": null, + "tooltip": null, + "value": "100%" + } + }, + "f1355871cc6f4dd4b50d9df5af20e5c8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/website/static/img/create_gcp_svc.png b/website/static/img/create_gcp_svc.png new file mode 100644 index 000000000000..aefd4af762fe --- /dev/null +++ b/website/static/img/create_gcp_svc.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7aa39b25ffcaba97cfd1a87cb1b44d45d3814241c83122f84408e03ad575b0 +size 101583 From 4e95630fa95111166d29d24fe47ffcdea059acbb Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 10 Jul 2024 15:12:42 -0700 Subject: [PATCH 31/57] [.Net] fix #2695 and #2884 (#3069) * add round robin orchestrator * add constructor for orchestrators * add tests * revert change * return single orchestrator * address comment --- dotnet/Directory.Build.props | 1 + .../Agent/AnthropicClientAgent.cs | 21 +- .../src/AutoGen.Anthropic/AnthropicClient.cs | 10 +- dotnet/src/AutoGen.Core/Agent/IAgent.cs | 6 +- .../Extension/GroupChatExtension.cs | 3 +- .../src/AutoGen.Core/GroupChat/GroupChat.cs | 78 +++-- .../{ => GroupChat}/IGroupChat.cs | 0 .../GroupChat/RoundRobinGroupChat.cs | 71 +--- .../Orchestrator/IOrchestrator.cs | 28 ++ .../Orchestrator/RolePlayOrchestrator.cs | 116 ++++++ .../Orchestrator/RoundRobinOrchestrator.cs | 45 +++ .../Orchestrator/WorkflowOrchestrator.cs | 53 +++ .../Agent/MistralClientAgent.cs | 1 + .../DTOs/ChatCompletionRequest.cs | 3 + .../AnthropicClientAgentTest.cs | 24 ++ .../test/AutoGen.Tests/AutoGen.Tests.csproj | 1 + .../Orchestrator/RolePlayOrchestratorTests.cs | 331 ++++++++++++++++++ .../RoundRobinOrchestratorTests.cs | 103 ++++++ .../Orchestrator/WorkflowOrchestratorTests.cs | 112 ++++++ 19 files changed, 901 insertions(+), 106 deletions(-) rename dotnet/src/AutoGen.Core/{ => GroupChat}/IGroupChat.cs (100%) create mode 100644 dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs create mode 100644 dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs create mode 100644 dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs create mode 100644 dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs create mode 100644 dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs create mode 100644 dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs create mode 100644 dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props index 4b3e9441f1ee..29e40fff384c 100644 --- a/dotnet/Directory.Build.props +++ b/dotnet/Directory.Build.props @@ -31,6 +31,7 @@ + diff --git a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs index bf05ee974445..a3ecc1ccef65 100644 --- a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs +++ b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs @@ -67,7 +67,8 @@ private ChatCompletionRequest CreateParameters(IEnumerable messages, G Stream = shouldStream, Temperature = (decimal?)options?.Temperature ?? _temperature, Tools = _tools?.ToList(), - ToolChoice = _toolChoice ?? ToolChoice.Auto + ToolChoice = _toolChoice ?? (_tools is { Length: > 0 } ? ToolChoice.Auto : null), + StopSequences = options?.StopSequence?.ToArray(), }; chatCompletionRequest.Messages = BuildMessages(messages); @@ -95,6 +96,22 @@ private List BuildMessages(IEnumerable messages) } } - return chatMessages; + // merge messages with the same role + // fixing #2884 + var mergedMessages = chatMessages.Aggregate(new List(), (acc, message) => + { + if (acc.Count > 0 && acc.Last().Role == message.Role) + { + acc.Last().Content.AddRange(message.Content); + } + else + { + acc.Add(message); + } + + return acc; + }); + + return mergedMessages; } } diff --git a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs index babcd5302aac..dd35638c4f3a 100644 --- a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs +++ b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // AnthropicClient.cs using System; @@ -90,13 +90,7 @@ public async IAsyncEnumerable StreamingChatCompletionsAs { var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), - cancellationToken: cancellationToken); - - if (res == null) - { - throw new Exception("Failed to deserialize response"); - } - + cancellationToken: cancellationToken) ?? throw new Exception("Failed to deserialize response"); if (res.Delta?.Type == "input_json_delta" && !string.IsNullOrEmpty(res.Delta.PartialJson) && currentEvent.ContentBlock != null) { diff --git a/dotnet/src/AutoGen.Core/Agent/IAgent.cs b/dotnet/src/AutoGen.Core/Agent/IAgent.cs index b9149008480d..34a31055d1bf 100644 --- a/dotnet/src/AutoGen.Core/Agent/IAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/IAgent.cs @@ -7,10 +7,14 @@ using System.Threading.Tasks; namespace AutoGen.Core; -public interface IAgent + +public interface IAgentMetaInformation { public string Name { get; } +} +public interface IAgent : IAgentMetaInformation +{ /// /// Generate reply /// diff --git a/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs b/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs index e3e44622c817..45728023b96b 100644 --- a/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs +++ b/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs @@ -100,8 +100,7 @@ internal static IEnumerable ProcessConversationsForRolePlay( var msg = @$"From {x.From}: {x.GetContent()} -round # - {i}"; +round # {i}"; return new TextMessage(Role.User, content: msg); }); diff --git a/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs b/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs index 5e82931ab658..57e15c18ca62 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs @@ -15,6 +15,7 @@ public class GroupChat : IGroupChat private List agents = new List(); private IEnumerable initializeMessages = new List(); private Graph? workflow = null; + private readonly IOrchestrator orchestrator; public IEnumerable? Messages { get; private set; } @@ -36,6 +37,37 @@ public GroupChat( this.initializeMessages = initializeMessages ?? new List(); this.workflow = workflow; + if (admin is not null) + { + this.orchestrator = new RolePlayOrchestrator(admin, workflow); + } + else if (workflow is not null) + { + this.orchestrator = new WorkflowOrchestrator(workflow); + } + else + { + this.orchestrator = new RoundRobinOrchestrator(); + } + + this.Validation(); + } + + /// + /// Create a group chat which uses the to decide the next speaker(s). + /// + /// + /// + /// + public GroupChat( + IEnumerable members, + IOrchestrator orchestrator, + IEnumerable? initializeMessages = null) + { + this.agents = members.ToList(); + this.initializeMessages = initializeMessages ?? new List(); + this.orchestrator = orchestrator; + this.Validation(); } @@ -64,12 +96,6 @@ private void Validation() throw new Exception("All agents in the workflow must be in the group chat."); } } - - // must provide one of admin or workflow - if (this.admin == null && this.workflow == null) - { - throw new Exception("Must provide one of admin or workflow."); - } } /// @@ -81,6 +107,7 @@ private void Validation() /// current speaker /// conversation history /// next speaker. + [Obsolete("Please use RolePlayOrchestrator or WorkflowOrchestrator")] public async Task SelectNextSpeakerAsync(IAgent currentSpeaker, IEnumerable conversationHistory) { var agentNames = this.agents.Select(x => x.Name).ToList(); @@ -140,37 +167,40 @@ public void AddInitializeMessage(IMessage message) } public async Task> CallAsync( - IEnumerable? conversationWithName = null, + IEnumerable? chatHistory = null, int maxRound = 10, CancellationToken ct = default) { var conversationHistory = new List(); - if (conversationWithName != null) + conversationHistory.AddRange(this.initializeMessages); + if (chatHistory != null) { - conversationHistory.AddRange(conversationWithName); + conversationHistory.AddRange(chatHistory); } + var roundLeft = maxRound; - var lastSpeaker = conversationHistory.LastOrDefault()?.From switch + while (roundLeft > 0) { - null => this.agents.First(), - _ => this.agents.FirstOrDefault(x => x.Name == conversationHistory.Last().From) ?? throw new Exception("The agent is not in the group chat"), - }; - var round = 0; - while (round < maxRound) - { - var currentSpeaker = await this.SelectNextSpeakerAsync(lastSpeaker, conversationHistory); - var processedConversation = this.ProcessConversationForAgent(this.initializeMessages, conversationHistory); - var result = await currentSpeaker.GenerateReplyAsync(processedConversation) ?? throw new Exception("No result is returned."); + var orchestratorContext = new OrchestrationContext + { + Candidates = this.agents, + ChatHistory = conversationHistory, + }; + var nextSpeaker = await this.orchestrator.GetNextSpeakerAsync(orchestratorContext, ct); + if (nextSpeaker == null) + { + break; + } + + var result = await nextSpeaker.GenerateReplyAsync(conversationHistory, cancellationToken: ct); conversationHistory.Add(result); - // if message is terminate message, then terminate the conversation - if (result?.IsGroupChatTerminateMessage() ?? false) + if (result.IsGroupChatTerminateMessage()) { - break; + return conversationHistory; } - lastSpeaker = currentSpeaker; - round++; + roundLeft--; } return conversationHistory; diff --git a/dotnet/src/AutoGen.Core/IGroupChat.cs b/dotnet/src/AutoGen.Core/GroupChat/IGroupChat.cs similarity index 100% rename from dotnet/src/AutoGen.Core/IGroupChat.cs rename to dotnet/src/AutoGen.Core/GroupChat/IGroupChat.cs diff --git a/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs b/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs index b8de89b834fe..b95cd1958fc5 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs @@ -3,9 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; namespace AutoGen.Core; @@ -25,76 +22,12 @@ public SequentialGroupChat(IEnumerable agents, List? initializ /// /// A group chat that allows agents to talk in a round-robin manner. /// -public class RoundRobinGroupChat : IGroupChat +public class RoundRobinGroupChat : GroupChat { - private readonly List agents = new List(); - private readonly List initializeMessages = new List(); - public RoundRobinGroupChat( IEnumerable agents, List? initializeMessages = null) + : base(agents, initializeMessages: initializeMessages) { - this.agents.AddRange(agents); - this.initializeMessages = initializeMessages ?? new List(); - } - - /// - public void AddInitializeMessage(IMessage message) - { - this.SendIntroduction(message); - } - - public async Task> CallAsync( - IEnumerable? conversationWithName = null, - int maxRound = 10, - CancellationToken ct = default) - { - var conversationHistory = new List(); - if (conversationWithName != null) - { - conversationHistory.AddRange(conversationWithName); - } - - var lastSpeaker = conversationHistory.LastOrDefault()?.From switch - { - null => this.agents.First(), - _ => this.agents.FirstOrDefault(x => x.Name == conversationHistory.Last().From) ?? throw new Exception("The agent is not in the group chat"), - }; - var round = 0; - while (round < maxRound) - { - var currentSpeaker = this.SelectNextSpeaker(lastSpeaker); - var processedConversation = this.ProcessConversationForAgent(this.initializeMessages, conversationHistory); - var result = await currentSpeaker.GenerateReplyAsync(processedConversation) ?? throw new Exception("No result is returned."); - conversationHistory.Add(result); - - // if message is terminate message, then terminate the conversation - if (result?.IsGroupChatTerminateMessage() ?? false) - { - break; - } - - lastSpeaker = currentSpeaker; - round++; - } - - return conversationHistory; - } - - public void SendIntroduction(IMessage message) - { - this.initializeMessages.Add(message); - } - - private IAgent SelectNextSpeaker(IAgent currentSpeaker) - { - var index = this.agents.IndexOf(currentSpeaker); - if (index == -1) - { - throw new ArgumentException("The agent is not in the group chat", nameof(currentSpeaker)); - } - - var nextIndex = (index + 1) % this.agents.Count; - return this.agents[nextIndex]; } } diff --git a/dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs new file mode 100644 index 000000000000..777834871f65 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs @@ -0,0 +1,28 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// IOrchestrator.cs + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +public class OrchestrationContext +{ + public IEnumerable Candidates { get; set; } = Array.Empty(); + + public IEnumerable ChatHistory { get; set; } = Array.Empty(); +} + +public interface IOrchestrator +{ + /// + /// Return the next agent as the next speaker. return null if no agent is selected. + /// + /// orchestration context, such as candidate agents and chat history. + /// cancellation token + public Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs new file mode 100644 index 000000000000..6798f23f2df8 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs @@ -0,0 +1,116 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// RolePlayOrchestrator.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +public class RolePlayOrchestrator : IOrchestrator +{ + private readonly IAgent admin; + private readonly Graph? workflow = null; + public RolePlayOrchestrator(IAgent admin, Graph? workflow = null) + { + this.admin = admin; + this.workflow = workflow; + } + + public async Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default) + { + var candidates = context.Candidates.ToList(); + + if (candidates.Count == 0) + { + return null; + } + + if (candidates.Count == 1) + { + return candidates.First(); + } + + // if there's a workflow + // and the next available agent from the workflow is in the group chat + // then return the next agent from the workflow + if (this.workflow != null) + { + var lastMessage = context.ChatHistory.LastOrDefault(); + if (lastMessage == null) + { + return null; + } + var currentSpeaker = candidates.First(candidates => candidates.Name == lastMessage.From); + var nextAgents = await this.workflow.TransitToNextAvailableAgentsAsync(currentSpeaker, context.ChatHistory); + nextAgents = nextAgents.Where(nextAgent => candidates.Any(candidate => candidate.Name == nextAgent.Name)); + candidates = nextAgents.ToList(); + if (!candidates.Any()) + { + return null; + } + + if (candidates is { Count: 1 }) + { + return candidates.First(); + } + } + + // In this case, since there are more than one available agents from the workflow for the next speaker + // the admin will be invoked to decide the next speaker + var agentNames = candidates.Select(candidate => candidate.Name); + var rolePlayMessage = new TextMessage(Role.User, + content: $@"You are in a role play game. Carefully read the conversation history and carry on the conversation. +The available roles are: +{string.Join(",", agentNames)} + +Each message will start with 'From name:', e.g: +From {agentNames.First()}: +//your message//."); + + var chatHistoryWithName = this.ProcessConversationsForRolePlay(context.ChatHistory); + var messages = new IMessage[] { rolePlayMessage }.Concat(chatHistoryWithName); + + var response = await this.admin.GenerateReplyAsync( + messages: messages, + options: new GenerateReplyOptions + { + Temperature = 0, + MaxToken = 128, + StopSequence = [":"], + Functions = null, + }, + cancellationToken: cancellationToken); + + var name = response.GetContent() ?? throw new Exception("No name is returned."); + + // remove From + name = name!.Substring(5); + var candidate = candidates.FirstOrDefault(x => x.Name!.ToLower() == name.ToLower()); + + if (candidate != null) + { + return candidate; + } + + var errorMessage = $"The response from admin is {name}, which is either not in the candidates list or not in the correct format."; + throw new Exception(errorMessage); + } + + private IEnumerable ProcessConversationsForRolePlay(IEnumerable messages) + { + return messages.Select((x, i) => + { + var msg = @$"From {x.From}: +{x.GetContent()} + +round # {i}"; + + return new TextMessage(Role.User, content: msg); + }); + } +} diff --git a/dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs new file mode 100644 index 000000000000..0f8b8e483c63 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs @@ -0,0 +1,45 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// RoundRobinOrchestrator.cs + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +/// +/// Return the next agent in a round-robin fashion. +/// +/// If the last message is from one of the candidates, the next agent will be the next candidate in the list. +/// +/// +/// Otherwise, no agent will be selected. In this case, the orchestrator will return an empty list. +/// +/// +/// This orchestrator always return a single agent. +/// +/// +public class RoundRobinOrchestrator : IOrchestrator +{ + public async Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default) + { + var lastMessage = context.ChatHistory.LastOrDefault(); + + if (lastMessage == null) + { + return null; + } + + var candidates = context.Candidates.ToList(); + var lastAgentIndex = candidates.FindIndex(a => a.Name == lastMessage.From); + if (lastAgentIndex == -1) + { + return null; + } + + var nextAgentIndex = (lastAgentIndex + 1) % candidates.Count; + return candidates[nextAgentIndex]; + } +} diff --git a/dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs new file mode 100644 index 000000000000..b84850a07c75 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs @@ -0,0 +1,53 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// WorkflowOrchestrator.cs + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +public class WorkflowOrchestrator : IOrchestrator +{ + private readonly Graph workflow; + + public WorkflowOrchestrator(Graph workflow) + { + this.workflow = workflow; + } + + public async Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default) + { + var lastMessage = context.ChatHistory.LastOrDefault(); + if (lastMessage == null) + { + return null; + } + + var candidates = context.Candidates.ToList(); + var currentSpeaker = candidates.FirstOrDefault(candidates => candidates.Name == lastMessage.From); + + if (currentSpeaker == null) + { + return null; + } + var nextAgents = await this.workflow.TransitToNextAvailableAgentsAsync(currentSpeaker, context.ChatHistory); + nextAgents = nextAgents.Where(nextAgent => candidates.Any(candidate => candidate.Name == nextAgent.Name)); + candidates = nextAgents.ToList(); + if (!candidates.Any()) + { + return null; + } + + if (candidates is { Count: 1 }) + { + return candidates.First(); + } + else + { + throw new System.Exception("There are more than one available agents from the workflow for the next speaker."); + } + } +} diff --git a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs index ac144854fac7..db14d68a1217 100644 --- a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs +++ b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs @@ -97,6 +97,7 @@ private ChatCompletionRequest BuildChatRequest(IEnumerable messages, G var chatHistory = BuildChatHistory(messages); var chatRequest = new ChatCompletionRequest(model: _model, messages: chatHistory.ToList(), temperature: options?.Temperature, randomSeed: _randomSeed) { + Stop = options?.StopSequence, MaxTokens = options?.MaxToken, ResponseFormat = _jsonOutput ? new ResponseFormat() { ResponseFormatType = "json_object" } : null, }; diff --git a/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs index 71a084673f13..affe2bb6dcc3 100644 --- a/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs +++ b/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs @@ -105,6 +105,9 @@ public class ChatCompletionRequest [JsonPropertyName("random_seed")] public int? RandomSeed { get; set; } + [JsonPropertyName("stop")] + public string[]? Stop { get; set; } + [JsonPropertyName("tools")] public List? Tools { get; set; } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs index 49cbb54af318..552408f1d056 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs @@ -32,6 +32,30 @@ public async Task AnthropicAgentChatCompletionTestAsync() reply.From.Should().Be(agent.Name); } + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentMergeMessageWithSameRoleTests() + { + // this test is added to fix issue #2884 + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant that convert user message to upper case") + .RegisterMessageConnector(); + + var uppCaseMessage = new TextMessage(Role.User, "abcdefg"); + var anotherUserMessage = new TextMessage(Role.User, "hijklmn"); + var assistantMessage = new TextMessage(Role.Assistant, "opqrst"); + var anotherAssistantMessage = new TextMessage(Role.Assistant, "uvwxyz"); + var yetAnotherUserMessage = new TextMessage(Role.User, "123456"); + + // just make sure it doesn't throw exception + var reply = await agent.SendAsync(chatHistory: [uppCaseMessage, anotherUserMessage, assistantMessage, anotherAssistantMessage, yetAnotherUserMessage]); + reply.GetContent().Should().NotBeNull(); + } + [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicAgentTestProcessImageAsync() { diff --git a/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj b/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj index 4def281ed7b4..3dc669b5edd8 100644 --- a/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj +++ b/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs new file mode 100644 index 000000000000..eea0a90c1caf --- /dev/null +++ b/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs @@ -0,0 +1,331 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// RolePlayOrchestratorTests.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using AutoGen.Anthropic; +using AutoGen.Anthropic.Extensions; +using AutoGen.Anthropic.Utils; +using AutoGen.Gemini; +using AutoGen.Mistral; +using AutoGen.Mistral.Extension; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using FluentAssertions; +using Moq; +using Xunit; + +namespace AutoGen.Tests; + +public class RolePlayOrchestratorTests +{ + [Fact] + public async Task ItReturnNextSpeakerTestAsync() + { + var admin = Mock.Of(); + Mock.Get(admin).Setup(x => x.Name).Returns("Admin"); + Mock.Get(admin).Setup(x => x.GenerateReplyAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Callback, GenerateReplyOptions, CancellationToken>((messages, option, _) => + { + // verify prompt + var rolePlayPrompt = messages.First().GetContent(); + rolePlayPrompt.Should().Contain("You are in a role play game. Carefully read the conversation history and carry on the conversation"); + rolePlayPrompt.Should().Contain("The available roles are:"); + rolePlayPrompt.Should().Contain("Alice,Bob"); + rolePlayPrompt.Should().Contain("From Alice:"); + option.StopSequence.Should().BeEquivalentTo([":"]); + option.Temperature.Should().Be(0); + option.MaxToken.Should().Be(128); + option.Functions.Should().BeNull(); + }) + .ReturnsAsync(new TextMessage(Role.Assistant, "From Alice")); + + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = [], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(alice); + } + + [Fact] + public async Task ItReturnNullWhenNoCandidateIsAvailableAsync() + { + var admin = Mock.Of(); + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [], + ChatHistory = [], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().BeNull(); + } + + [Fact] + public async Task ItReturnCandidateWhenOnlyOneCandidateIsAvailableAsync() + { + var admin = Mock.Of(); + var alice = new EchoAgent("Alice"); + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [alice], + ChatHistory = [], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(alice); + } + + [Fact] + public async Task ItThrowExceptionWhenAdminFailsToFollowPromptAsync() + { + var admin = Mock.Of(); + Mock.Get(admin).Setup(x => x.Name).Returns("Admin"); + Mock.Get(admin).Setup(x => x.GenerateReplyAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new TextMessage(Role.Assistant, "I don't know")); // admin fails to follow the prompt and returns an invalid message + + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = [], + }; + + var action = async () => await orchestrator.GetNextSpeakerAsync(context); + + await action.Should().ThrowAsync() + .WithMessage("The response from admin is 't know, which is either not in the candidates list or not in the correct format."); + } + + [Fact] + public async Task ItSelectNextSpeakerFromWorkflowIfProvided() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(bob, charlie)); + workflow.AddTransition(Transition.Create(charlie, alice)); + + var admin = Mock.Of(); + var orchestrator = new RolePlayOrchestrator(admin, workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob, charlie], + ChatHistory = + [ + new TextMessage(Role.User, "Hello, Bob", from: "Alice"), + ], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(bob); + } + + [Fact] + public async Task ItReturnNullIfNoAvailableAgentFromWorkflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + workflow.AddTransition(Transition.Create(alice, bob)); + + var admin = Mock.Of(); + var orchestrator = new RolePlayOrchestrator(admin, workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = + [ + new TextMessage(Role.User, "Hello, Alice", from: "Bob"), + ], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().BeNull(); + } + + [Fact] + public async Task ItUseCandidatesFromWorflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(alice, charlie)); + + var admin = Mock.Of(); + Mock.Get(admin).Setup(x => x.GenerateReplyAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Callback, GenerateReplyOptions, CancellationToken>((messages, option, _) => + { + messages.First().IsSystemMessage().Should().BeTrue(); + + // verify prompt + var rolePlayPrompt = messages.First().GetContent(); + rolePlayPrompt.Should().Contain("Bob,Charlie"); + rolePlayPrompt.Should().Contain("From Bob:"); + option.StopSequence.Should().BeEquivalentTo([":"]); + option.Temperature.Should().Be(0); + option.MaxToken.Should().Be(128); + option.Functions.Should().BeEmpty(); + }) + .ReturnsAsync(new TextMessage(Role.Assistant, "From Bob")); + var orchestrator = new RolePlayOrchestrator(admin, workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = + [ + new TextMessage(Role.User, "Hello, Bob", from: "Alice"), + ], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(bob); + } + + [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] + public async Task GPT_3_5_CoderReviewerRunnerTestAsync() + { + var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new Exception("Please set AZURE_OPENAI_ENDPOINT environment variable."); + var key = Environment.GetEnvironmentVariable("AZURE_OPENAI_API_KEY") ?? throw new Exception("Please set AZURE_OPENAI_API_KEY environment variable."); + var deployName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOY_NAME") ?? throw new Exception("Please set AZURE_OPENAI_DEPLOY_NAME environment variable."); + var openaiClient = new OpenAIClient(new Uri(endpoint), new Azure.AzureKeyCredential(key)); + var openAIChatAgent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "assistant", + modelName: deployName) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(openAIChatAgent); + } + + [ApiKeyFact("GOOGLE_GEMINI_API_KEY")] + public async Task GoogleGemini_1_5_flash_001_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("GOOGLE_GEMINI_API_KEY") ?? throw new InvalidOperationException("GOOGLE_GEMINI_API_KEY is not set"); + var geminiAgent = new GeminiChatAgent( + name: "gemini", + model: "gemini-1.5-flash-001", + apiKey: apiKey) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(geminiAgent); + } + + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task Claude3_Haiku_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? throw new Exception("Please set ANTHROPIC_API_KEY environment variable."); + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, apiKey); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant that convert user message to upper case") + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(agent); + } + + [ApiKeyFact("MISTRAL_API_KEY")] + public async Task Mistra_7b_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("MISTRAL_API_KEY") ?? throw new InvalidOperationException("MISTRAL_API_KEY is not set."); + var client = new MistralClient(apiKey: apiKey); + + var agent = new MistralClientAgent( + client: client, + name: "MistralClientAgent", + model: "open-mistral-7b") + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(agent); + } + + /// + /// This test is to mimic the conversation among coder, reviewer and runner. + /// The coder will write the code, the reviewer will review the code, and the runner will run the code. + /// + /// + /// + public async Task CoderReviewerRunnerTestAsync(IAgent admin) + { + var coder = new EchoAgent("Coder"); + var reviewer = new EchoAgent("Reviewer"); + var runner = new EchoAgent("Runner"); + var user = new EchoAgent("User"); + var initializeMessage = new List + { + new TextMessage(Role.User, "Hello, I am user, I will provide the coding task, please write the code first, then review and run it", from: "User"), + new TextMessage(Role.User, "Hello, I am coder, I will write the code", from: "Coder"), + new TextMessage(Role.User, "Hello, I am reviewer, I will review the code", from: "Reviewer"), + new TextMessage(Role.User, "Hello, I am runner, I will run the code", from: "Runner"), + new TextMessage(Role.User, "how to print 'hello world' using C#", from: user.Name), + }; + + var chatHistory = new List() + { + new TextMessage(Role.User, """ + ```csharp + Console.WriteLine("Hello World"); + ``` + """, from: coder.Name), + new TextMessage(Role.User, "The code looks good", from: reviewer.Name), + new TextMessage(Role.User, "The code runs successfully, the output is 'Hello World'", from: runner.Name), + }; + + var orchestrator = new RolePlayOrchestrator(admin); + foreach (var message in chatHistory) + { + var context = new OrchestrationContext + { + Candidates = [coder, reviewer, runner, user], + ChatHistory = initializeMessage, + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker!.Name.Should().Be(message.From); + initializeMessage.Add(message); + } + + // the last next speaker should be the user + var lastSpeaker = await orchestrator.GetNextSpeakerAsync(new OrchestrationContext + { + Candidates = [coder, reviewer, runner, user], + ChatHistory = initializeMessage, + }); + + lastSpeaker!.Name.Should().Be(user.Name); + } +} diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs new file mode 100644 index 000000000000..e14bf85cf215 --- /dev/null +++ b/dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs @@ -0,0 +1,103 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// RoundRobinOrchestratorTests.cs + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Xunit; + +namespace AutoGen.Tests; + +public class RoundRobinOrchestratorTests +{ + [Fact] + public async Task ItReturnNextAgentAsync() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List + { + new EchoAgent("Alice"), + new EchoAgent("Bob"), + new EchoAgent("Charlie"), + }, + }; + + var messages = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + new TextMessage(Role.User, "Hello, Bob", from: "Bob"), + new TextMessage(Role.User, "Hello, Charlie", from: "Charlie"), + }; + + var expected = new List { "Bob", "Charlie", "Alice" }; + + var zip = messages.Zip(expected); + + foreach (var (msg, expect) in zip) + { + context.ChatHistory = [msg]; + var nextSpeaker = await orchestrator.GetNextSpeakerAsync(context); + Assert.Equal(expect, nextSpeaker!.Name); + } + } + + [Fact] + public async Task ItReturnNullIfNoCandidates() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List(), + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + }, + }; + + var result = await orchestrator.GetNextSpeakerAsync(context); + Assert.Null(result); + } + + [Fact] + public async Task ItReturnNullIfLastMessageIsNotFromCandidates() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List + { + new EchoAgent("Alice"), + new EchoAgent("Bob"), + new EchoAgent("Charlie"), + }, + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, David", from: "David"), + }, + }; + + var result = await orchestrator.GetNextSpeakerAsync(context); + result.Should().BeNull(); + } + + [Fact] + public async Task ItReturnEmptyListIfNoChatHistory() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List + { + new EchoAgent("Alice"), + new EchoAgent("Bob"), + new EchoAgent("Charlie"), + }, + }; + + var result = await orchestrator.GetNextSpeakerAsync(context); + result.Should().BeNull(); + } +} diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs new file mode 100644 index 000000000000..6599566a4466 --- /dev/null +++ b/dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs @@ -0,0 +1,112 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// WorkflowOrchestratorTests.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Xunit; + +namespace AutoGen.Tests; + +public class WorkflowOrchestratorTests +{ + [Fact] + public async Task ItReturnNextAgentAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(bob, charlie)); + workflow.AddTransition(Transition.Create(charlie, alice)); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob, charlie] + }; + + var messages = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + new TextMessage(Role.User, "Hello, Bob", from: "Bob"), + new TextMessage(Role.User, "Hello, Charlie", from: "Charlie"), + }; + + var expected = new List { "Bob", "Charlie", "Alice" }; + + var zip = messages.Zip(expected); + + foreach (var (msg, expect) in zip) + { + context.ChatHistory = [msg]; + var result = await orchestrator.GetNextSpeakerAsync(context); + Assert.Equal(expect, result!.Name); + } + } + + [Fact] + public async Task ItReturnNullIfNoCandidates() + { + var workflow = new Graph(); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = new List(), + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + }, + }; + + var nextAgent = await orchestrator.GetNextSpeakerAsync(context); + nextAgent.Should().BeNull(); + } + + [Fact] + public async Task ItReturnNullIfNoAgentIsAvailableFromWorkflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + workflow.AddTransition(Transition.Create(alice, bob)); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Bob", from: "Bob"), + }, + }; + + var nextSpeaker = await orchestrator.GetNextSpeakerAsync(context); + nextSpeaker.Should().BeNull(); + } + + [Fact] + public async Task ItThrowExceptionWhenMoreThanOneAvailableAgentsFromWorkflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(alice, charlie)); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob, charlie], + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Bob", from: "Alice"), + }, + }; + + var action = async () => await orchestrator.GetNextSpeakerAsync(context); + + await action.Should().ThrowExactlyAsync().WithMessage("There are more than one available agents from the workflow for the next speaker."); + } +} From b021e4454085f806e17ab5731d8019c56a67f3e6 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Thu, 11 Jul 2024 09:34:15 -0700 Subject: [PATCH 32/57] [.Net] Agent as service: Run an `IAgent` as openai chat completion endpoint (#2633) * update * add test * clean up * update * Delete dotnet/src/AutoGen.Server/AutoGen.Service.csproj.user * implement streaming * add sample project * rename AutoGen.Service to AutoGen.WebAPI * rename AutoGen.Service to AutoGen.WebAPI --- dotnet/AutoGen.sln | 26 ++- dotnet/eng/Version.props | 1 + .../AutoGen.WebAPI.Sample.csproj | 13 ++ .../sample/AutoGen.WebAPI.Sample/Program.cs | 45 +++++ .../src/AutoGen.WebAPI/AutoGen.WebAPI.csproj | 16 ++ dotnet/src/AutoGen.WebAPI/Extension.cs | 24 +++ .../Converter/OpenAIMessageConverter.cs | 56 +++++++ .../OpenAI/DTO/OpenAIAssistantMessage.cs | 21 +++ .../OpenAI/DTO/OpenAIChatCompletion.cs | 30 ++++ .../OpenAI/DTO/OpenAIChatCompletionChoice.cs | 21 +++ .../OpenAI/DTO/OpenAIChatCompletionMessage.cs | 15 ++ .../OpenAI/DTO/OpenAIChatCompletionOption.cs | 33 ++++ .../OpenAI/DTO/OpenAIChatCompletionUsage.cs | 18 ++ .../OpenAI/DTO/OpenAIImageUrlObject.cs | 15 ++ .../OpenAI/DTO/OpenAIMessage.cs | 13 ++ .../OpenAI/DTO/OpenAIStreamOptions.cs | 12 ++ .../OpenAI/DTO/OpenAISystemMessage.cs | 18 ++ .../OpenAI/DTO/OpenAIToolCallObject.cs | 15 ++ .../OpenAI/DTO/OpenAIToolMessage.cs | 18 ++ .../OpenAI/DTO/OpenAIUserImageContent.cs | 15 ++ .../OpenAI/DTO/OpenAIUserMessage.cs | 18 ++ .../OpenAI/DTO/OpenAIUserMessageItem.cs | 12 ++ .../OpenAI/DTO/OpenAIUserMultiModalMessage.cs | 18 ++ .../OpenAI/DTO/OpenAIUserTextContent.cs | 15 ++ .../Service/OpenAIChatCompletionService.cs | 157 ++++++++++++++++++ .../OpenAIChatCompletionMiddleware.cs | 92 ++++++++++ .../AutoGen.WebAPI.Tests.csproj | 28 ++++ dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs | 45 +++++ .../OpenAIChatCompletionMiddlewareTests.cs | 82 +++++++++ 29 files changed, 889 insertions(+), 3 deletions(-) create mode 100644 dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj create mode 100644 dotnet/sample/AutoGen.WebAPI.Sample/Program.cs create mode 100644 dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj create mode 100644 dotnet/src/AutoGen.WebAPI/Extension.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs create mode 100644 dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs create mode 100644 dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj create mode 100644 dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs create mode 100644 dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs diff --git a/dotnet/AutoGen.sln b/dotnet/AutoGen.sln index 5fa215f0ce9c..1218cf129821 100644 --- a/dotnet/AutoGen.sln +++ b/dotnet/AutoGen.sln @@ -1,4 +1,3 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34322.80 @@ -33,6 +32,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Mistral", "src\Auto EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Mistral.Tests", "test\AutoGen.Mistral.Tests\AutoGen.Mistral.Tests.csproj", "{15441693-3659-4868-B6C1-B106F52FF3BA}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI", "src\AutoGen.WebAPI\AutoGen.WebAPI.csproj", "{257FFD71-08E5-40C7-AB04-6A81A78EB410}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI.Tests", "test\AutoGen.WebAPI.Tests\AutoGen.WebAPI.Tests.csproj", "{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.SemanticKernel.Tests", "test\AutoGen.SemanticKernel.Tests\AutoGen.SemanticKernel.Tests.csproj", "{1DFABC4A-8458-4875-8DCB-59F3802DAC65}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.OpenAI.Tests", "test\AutoGen.OpenAI.Tests\AutoGen.OpenAI.Tests.csproj", "{D36A85F9-C172-487D-8192-6BFE5D05B4A7}" @@ -61,7 +64,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Gemini.Sample", "sa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.AotCompatibility.Tests", "test\AutoGen.AotCompatibility.Tests\AutoGen.AotCompatibility.Tests.csproj", "{6B82F26D-5040-4453-B21B-C8D1F913CE4C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.OpenAI.Sample", "sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj", "{0E635268-351C-4A6B-A28D-593D868C2CA4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.OpenAI.Sample", "sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj", "{0E635268-351C-4A6B-A28D-593D868C2CA4}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI.Sample", "sample\AutoGen.WebAPI.Sample\AutoGen.WebAPI.Sample.csproj", "{12079C18-A519-403F-BBFD-200A36A0C083}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -117,6 +122,14 @@ Global {15441693-3659-4868-B6C1-B106F52FF3BA}.Debug|Any CPU.Build.0 = Debug|Any CPU {15441693-3659-4868-B6C1-B106F52FF3BA}.Release|Any CPU.ActiveCfg = Release|Any CPU {15441693-3659-4868-B6C1-B106F52FF3BA}.Release|Any CPU.Build.0 = Release|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Debug|Any CPU.Build.0 = Debug|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Release|Any CPU.ActiveCfg = Release|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Release|Any CPU.Build.0 = Release|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Release|Any CPU.Build.0 = Release|Any CPU {1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Debug|Any CPU.Build.0 = Debug|Any CPU {1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -177,6 +190,10 @@ Global {0E635268-351C-4A6B-A28D-593D868C2CA4}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.Build.0 = Release|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Debug|Any CPU.Build.0 = Debug|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Release|Any CPU.ActiveCfg = Release|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -194,6 +211,8 @@ Global {63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC} = {18BF8DD7-0585-48BF-8F97-AD333080CE06} {6585D1A4-3D97-4D76-A688-1933B61AEB19} = {18BF8DD7-0585-48BF-8F97-AD333080CE06} {15441693-3659-4868-B6C1-B106F52FF3BA} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} + {257FFD71-08E5-40C7-AB04-6A81A78EB410} = {18BF8DD7-0585-48BF-8F97-AD333080CE06} + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {1DFABC4A-8458-4875-8DCB-59F3802DAC65} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {D36A85F9-C172-487D-8192-6BFE5D05B4A7} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {B61388CA-DC73-4B7F-A7B2-7B9A86C9229E} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} @@ -209,8 +228,9 @@ Global {19679B75-CE3A-4DF0-A3F0-CA369D2760A4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} {6B82F26D-5040-4453-B21B-C8D1F913CE4C} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {0E635268-351C-4A6B-A28D-593D868C2CA4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} + {12079C18-A519-403F-BBFD-200A36A0C083} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B} EndGlobalSection -EndGlobal \ No newline at end of file +EndGlobal diff --git a/dotnet/eng/Version.props b/dotnet/eng/Version.props index 0b8dcaa565cb..572bb22e4676 100644 --- a/dotnet/eng/Version.props +++ b/dotnet/eng/Version.props @@ -12,6 +12,7 @@ 17.7.0 1.0.0-beta.24229.4 8.0.0 + 8.0.4 3.0.0 4.3.0.2 diff --git a/dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj b/dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj new file mode 100644 index 000000000000..41f3b7d1d381 --- /dev/null +++ b/dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj @@ -0,0 +1,13 @@ +๏ปฟ + + + net8.0 + enable + enable + + + + + + + diff --git a/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs b/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs new file mode 100644 index 000000000000..71b8b89219aa --- /dev/null +++ b/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs @@ -0,0 +1,45 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Program.cs + +using System.Runtime.CompilerServices; +using AutoGen.Core; +using AutoGen.Service; + +var alice = new DummyAgent("alice"); +var bob = new DummyAgent("bob"); + +var builder = WebApplication.CreateBuilder(args); +// Add services to the container. + +// run endpoint at port 5000 +builder.WebHost.UseUrls("http://localhost:5000"); +var app = builder.Build(); + +app.UseAgentAsOpenAIChatCompletionEndpoint(alice); +app.UseAgentAsOpenAIChatCompletionEndpoint(bob); + +app.Run(); + +public class DummyAgent : IStreamingAgent +{ + public DummyAgent(string name = "dummy") + { + Name = name; + } + + public string Name { get; } + + public async Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + { + return new TextMessage(Role.Assistant, $"I am dummy {this.Name}", this.Name); + } + + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var reply = $"I am dummy {this.Name}"; + foreach (var c in reply) + { + yield return new TextMessageUpdate(Role.Assistant, c.ToString(), this.Name); + }; + } +} diff --git a/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj b/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj new file mode 100644 index 000000000000..cd24d5930b36 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj @@ -0,0 +1,16 @@ +๏ปฟ + + + net6.0;net8.0 + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + diff --git a/dotnet/src/AutoGen.WebAPI/Extension.cs b/dotnet/src/AutoGen.WebAPI/Extension.cs new file mode 100644 index 000000000000..30fd8843174d --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/Extension.cs @@ -0,0 +1,24 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Extension.cs + +using AutoGen.Core; +using Microsoft.AspNetCore.Builder; + +namespace AutoGen.Service; + +public static class Extension +{ + /// + /// Serve the agent as an OpenAI chat completion endpoint using . + /// If the request path is /v1/chat/completions and model name is the same as the agent name, + /// the request will be handled by the agent. + /// otherwise, the request will be passed to the next middleware. + /// + /// application builder + /// + public static IApplicationBuilder UseAgentAsOpenAIChatCompletionEndpoint(this IApplicationBuilder app, IAgent agent) + { + var middleware = new OpenAIChatCompletionMiddleware(agent); + return app.Use(middleware.InvokeAsync); + } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs new file mode 100644 index 000000000000..78e7b8a6dd0a --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs @@ -0,0 +1,56 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIMessageConverter.cs + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIMessageConverter : JsonConverter +{ + public override OpenAIMessage Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using JsonDocument document = JsonDocument.ParseValue(ref reader); + var root = document.RootElement; + var role = root.GetProperty("role").GetString(); + var contentDocument = root.GetProperty("content"); + var isContentDocumentString = contentDocument.ValueKind == JsonValueKind.String; + switch (role) + { + case "system": + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "user" when isContentDocumentString: + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "user" when !isContentDocumentString: + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "assistant": + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "tool": + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + default: + throw new JsonException(); + } + } + + public override void Write(Utf8JsonWriter writer, OpenAIMessage value, JsonSerializerOptions options) + { + switch (value) + { + case OpenAISystemMessage systemMessage: + JsonSerializer.Serialize(writer, systemMessage, options); + break; + case OpenAIUserMessage userMessage: + JsonSerializer.Serialize(writer, userMessage, options); + break; + case OpenAIAssistantMessage assistantMessage: + JsonSerializer.Serialize(writer, assistantMessage, options); + break; + case OpenAIToolMessage toolMessage: + JsonSerializer.Serialize(writer, toolMessage, options); + break; + default: + throw new JsonException(); + } + } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs new file mode 100644 index 000000000000..62771d58a28c --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs @@ -0,0 +1,21 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIAssistantMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIAssistantMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "assistant"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("tool_calls")] + public OpenAIToolCallObject[]? ToolCalls { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs new file mode 100644 index 000000000000..c5fa3334c5b8 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs @@ -0,0 +1,30 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletion.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIChatCompletion +{ + [JsonPropertyName("id")] + public string? ID { get; set; } + + [JsonPropertyName("created")] + public long Created { get; set; } + + [JsonPropertyName("choices")] + public OpenAIChatCompletionChoice[]? Choices { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("system_fingerprint")] + public string? SystemFingerprint { get; set; } + + [JsonPropertyName("object")] + public string Object { get; set; } = "chat.completion"; + + [JsonPropertyName("usage")] + public OpenAIChatCompletionUsage? Usage { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs new file mode 100644 index 000000000000..82ba56a6adf2 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs @@ -0,0 +1,21 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionChoice.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIChatCompletionChoice +{ + [JsonPropertyName("finish_reason")] + public string? FinishReason { get; set; } + + [JsonPropertyName("index")] + public int Index { get; set; } + + [JsonPropertyName("message")] + public OpenAIChatCompletionMessage? Message { get; set; } + + [JsonPropertyName("delta")] + public OpenAIChatCompletionMessage? Delta { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs new file mode 100644 index 000000000000..8699523f792b --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs @@ -0,0 +1,15 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIChatCompletionMessage +{ + [JsonPropertyName("role")] + public string Role { get; } = "assistant"; + + [JsonPropertyName("content")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs new file mode 100644 index 000000000000..6e656ded8459 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs @@ -0,0 +1,33 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionOption.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIChatCompletionOption +{ + [JsonPropertyName("messages")] + public OpenAIMessage[]? Messages { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("max_tokens")] + public int? MaxTokens { get; set; } + + [JsonPropertyName("temperature")] + public float Temperature { get; set; } = 1; + + /// + /// If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only server-sent events as they become available, with the stream terminated by a data: [DONE] message + /// + [JsonPropertyName("stream")] + public bool? Stream { get; set; } = false; + + [JsonPropertyName("stream_options")] + public OpenAIStreamOptions? StreamOptions { get; set; } + + [JsonPropertyName("stop")] + public string[]? Stop { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs new file mode 100644 index 000000000000..299d62c9d9e0 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs @@ -0,0 +1,18 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionUsage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIChatCompletionUsage +{ + [JsonPropertyName("completion_tokens")] + public int CompletionTokens { get; set; } + + [JsonPropertyName("prompt_tokens")] + public int PromptTokens { get; set; } + + [JsonPropertyName("total_tokens")] + public int TotalTokens { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs new file mode 100644 index 000000000000..081620c414df --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs @@ -0,0 +1,15 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIImageUrlObject.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIImageUrlObject +{ + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("detail")] + public string? Detail { get; set; } = "auto"; +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs new file mode 100644 index 000000000000..3cad9ae36336 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs @@ -0,0 +1,13 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionOption.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +[JsonConverter(typeof(OpenAIMessageConverter))] +internal abstract class OpenAIMessage +{ + [JsonPropertyName("role")] + public abstract string? Role { get; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs new file mode 100644 index 000000000000..3c188b865b77 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs @@ -0,0 +1,12 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIStreamOptions +{ + [JsonPropertyName("include_usage")] + public bool? IncludeUsage { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs new file mode 100644 index 000000000000..57a671b8cdfa --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs @@ -0,0 +1,18 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAISystemMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAISystemMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "system"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs new file mode 100644 index 000000000000..7d003f2d797e --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs @@ -0,0 +1,15 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIToolCallObject.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIToolCallObject +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("arguments")] + public string? Arguments { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs new file mode 100644 index 000000000000..5a0bb968d8b5 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs @@ -0,0 +1,18 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIToolMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIToolMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "tool"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("tool_call_id")] + public string? ToolCallId { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs new file mode 100644 index 000000000000..c6ada9d8f73f --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs @@ -0,0 +1,15 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserImageContent.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIUserImageContent : OpenAIUserMessageItem +{ + [JsonPropertyName("type")] + public override string MessageType { get; } = "image"; + + [JsonPropertyName("image_url")] + public string? Url { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs new file mode 100644 index 000000000000..17cb529daf60 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs @@ -0,0 +1,18 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIUserMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "user"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs new file mode 100644 index 000000000000..af363c1c90ba --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs @@ -0,0 +1,12 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserMessageItem.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal abstract class OpenAIUserMessageItem +{ + [JsonPropertyName("type")] + public abstract string MessageType { get; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs new file mode 100644 index 000000000000..acb8279e7938 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs @@ -0,0 +1,18 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserMultiModalMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIUserMultiModalMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "user"; + + [JsonPropertyName("content")] + public OpenAIUserMessageItem[]? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs new file mode 100644 index 000000000000..fdb122253089 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs @@ -0,0 +1,15 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserTextContent.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.Service.OpenAI.DTO; + +internal class OpenAIUserTextContent : OpenAIUserMessageItem +{ + [JsonPropertyName("type")] + public override string MessageType { get; } = "text"; + + [JsonPropertyName("text")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs new file mode 100644 index 000000000000..b84f62c7ce20 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs @@ -0,0 +1,157 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionService.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AutoGen.Core; +using AutoGen.Service.OpenAI.DTO; + +namespace AutoGen.Server; + +internal class OpenAIChatCompletionService +{ + private readonly IAgent agent; + + public OpenAIChatCompletionService(IAgent agent) + { + this.agent = agent; + } + + public async Task GetChatCompletionAsync(OpenAIChatCompletionOption request) + { + var messages = this.ProcessMessages(request.Messages ?? Array.Empty()); + + var generateOption = this.ProcessReplyOptions(request); + + var reply = await this.agent.GenerateReplyAsync(messages, generateOption); + + var openAIChatCompletion = new OpenAIChatCompletion() + { + Created = DateTimeOffset.UtcNow.Ticks / TimeSpan.TicksPerMillisecond / 1000, + Model = this.agent.Name, + }; + + if (reply.GetContent() is string content) + { + var message = new OpenAIChatCompletionMessage() + { + Content = content, + }; + + var choice = new OpenAIChatCompletionChoice() + { + Message = message, + Index = 0, + FinishReason = "completed", + }; + + openAIChatCompletion.Choices = [choice]; + + return openAIChatCompletion; + } + + throw new NotImplementedException("Unsupported reply content type"); + } + + public async IAsyncEnumerable GetStreamingChatCompletionAsync(OpenAIChatCompletionOption request) + { + if (this.agent is IStreamingAgent streamingAgent) + { + var messages = this.ProcessMessages(request.Messages ?? Array.Empty()); + + var generateOption = this.ProcessReplyOptions(request); + + await foreach (var reply in streamingAgent.GenerateStreamingReplyAsync(messages, generateOption)) + { + var openAIChatCompletion = new OpenAIChatCompletion() + { + Created = DateTimeOffset.UtcNow.Ticks / TimeSpan.TicksPerMillisecond / 1000, + Model = this.agent.Name, + }; + + if (reply.GetContent() is string content) + { + var message = new OpenAIChatCompletionMessage() + { + Content = content, + }; + + var choice = new OpenAIChatCompletionChoice() + { + Delta = message, + Index = 0, + }; + + openAIChatCompletion.Choices = [choice]; + + yield return openAIChatCompletion; + } + else + { + throw new NotImplementedException("Unsupported reply content type"); + } + } + + var doneMessage = new OpenAIChatCompletion() + { + Created = DateTimeOffset.UtcNow.Ticks / TimeSpan.TicksPerMillisecond / 1000, + Model = this.agent.Name, + }; + + var doneChoice = new OpenAIChatCompletionChoice() + { + FinishReason = "stop", + Index = 0, + }; + + doneMessage.Choices = [doneChoice]; + + yield return doneMessage; + } + else + { + yield return await this.GetChatCompletionAsync(request); + } + } + + private IEnumerable ProcessMessages(IEnumerable messages) + { + return messages.Select(m => m switch + { + OpenAISystemMessage systemMessage when systemMessage.Content is string content => new TextMessage(Role.System, content, this.agent.Name), + OpenAIUserMessage userMessage when userMessage.Content is string content => new TextMessage(Role.User, content, this.agent.Name), + OpenAIAssistantMessage assistantMessage when assistantMessage.Content is string content => new TextMessage(Role.Assistant, content, this.agent.Name), + OpenAIUserMultiModalMessage userMultiModalMessage when userMultiModalMessage.Content is { Length: > 0 } => this.CreateMultiModaMessageFromOpenAIUserMultiModalMessage(userMultiModalMessage), + _ => throw new ArgumentException($"Unsupported message type {m.GetType()}") + }); + } + + private GenerateReplyOptions ProcessReplyOptions(OpenAIChatCompletionOption request) + { + return new GenerateReplyOptions() + { + Temperature = request.Temperature, + MaxToken = request.MaxTokens, + StopSequence = request.Stop, + }; + } + + private MultiModalMessage CreateMultiModaMessageFromOpenAIUserMultiModalMessage(OpenAIUserMultiModalMessage message) + { + if (message.Content is null) + { + throw new ArgumentNullException(nameof(message.Content)); + } + + IEnumerable items = message.Content.Select(item => item switch + { + OpenAIUserImageContent imageContent when imageContent.Url is string url => new ImageMessage(Role.User, url, this.agent.Name), + OpenAIUserTextContent textContent when textContent.Content is string content => new TextMessage(Role.User, content, this.agent.Name), + _ => throw new ArgumentException($"Unsupported content type {item.GetType()}") + }); + + return new MultiModalMessage(Role.User, items, this.agent.Name); + } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs b/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs new file mode 100644 index 000000000000..3563a6dc2bc3 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs @@ -0,0 +1,92 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMiddleware.cs + +using System.Text.Json; +using System.Threading.Tasks; +using AutoGen.Core; +using AutoGen.Server; +using AutoGen.Service.OpenAI.DTO; +using Microsoft.AspNetCore.Http; + +namespace AutoGen.Service; + +public class OpenAIChatCompletionMiddleware : Microsoft.AspNetCore.Http.IMiddleware +{ + private readonly IAgent _agent; + private readonly OpenAIChatCompletionService chatCompletionService; + + public OpenAIChatCompletionMiddleware(IAgent agent) + { + _agent = agent; + chatCompletionService = new OpenAIChatCompletionService(_agent); + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + // if HttpPost and path is /v1/chat/completions + // get the request body + // call chatCompletionService.GetChatCompletionAsync(request) + // return the response + + // else + // call next middleware + if (context.Request.Method == HttpMethods.Post && context.Request.Path == "/v1/chat/completions") + { + context.Request.EnableBuffering(); + var body = await context.Request.ReadFromJsonAsync(); + context.Request.Body.Position = 0; + if (body is null) + { + // return 400 Bad Request + context.Response.StatusCode = 400; + return; + } + + if (body.Model != _agent.Name) + { + await next(context); + return; + } + + if (body.Stream is true) + { + // Send as server side events + context.Response.Headers.Append("Content-Type", "text/event-stream"); + context.Response.Headers.Append("Cache-Control", "no-cache"); + context.Response.Headers.Append("Connection", "keep-alive"); + await foreach (var chatCompletion in chatCompletionService.GetStreamingChatCompletionAsync(body)) + { + if (chatCompletion?.Choices?[0].FinishReason is "stop") + { + // the stream is done + // send Data: [DONE]\n\n + await context.Response.WriteAsync("data: [DONE]\n\n"); + break; + } + else + { + // remove null + var option = new JsonSerializerOptions + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + var data = JsonSerializer.Serialize(chatCompletion, option); + await context.Response.WriteAsync($"data: {data}\n\n"); + } + } + + return; + } + else + { + var chatCompletion = await chatCompletionService.GetChatCompletionAsync(body); + await context.Response.WriteAsJsonAsync(chatCompletion); + return; + } + } + else + { + await next(context); + } + } +} diff --git a/dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj b/dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj new file mode 100644 index 000000000000..3a9caf38fc8e --- /dev/null +++ b/dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj @@ -0,0 +1,28 @@ +๏ปฟ + + + $(TestTargetFramework) + enable + enable + false + true + True + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs new file mode 100644 index 000000000000..64d48d2a00be --- /dev/null +++ b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs @@ -0,0 +1,45 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMiddlewareTests.cs + +using System.Runtime.CompilerServices; +using AutoGen.Core; + +namespace AutoGen.Service.Tests; + +public class EchoAgent : IStreamingAgent +{ + public EchoAgent(string name) + { + Name = name; + } + public string Name { get; } + + public async Task GenerateReplyAsync( + IEnumerable messages, + GenerateReplyOptions? options = null, + CancellationToken cancellationToken = default) + { + return messages.Last(); + } + + public async IAsyncEnumerable GenerateStreamingReplyAsync( + IEnumerable messages, + GenerateReplyOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var lastMessage = messages.LastOrDefault(); + if (lastMessage == null) + { + yield break; + } + + // return each character of the last message as a separate message + if (lastMessage.GetContent() is string content) + { + foreach (var c in content) + { + yield return new TextMessageUpdate(Role.Assistant, c.ToString(), this.Name); + } + } + } +} diff --git a/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs b/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs new file mode 100644 index 000000000000..bb0f6048e9a8 --- /dev/null +++ b/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs @@ -0,0 +1,82 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMiddlewareTests.cs + +using AutoGen.Core; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; +using FluentAssertions; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace AutoGen.Service.Tests; + +public class OpenAIChatCompletionMiddlewareTests +{ + [Fact] + public async Task ItReturnTextMessageWhenSendTextMessage() + { + var agent = new EchoAgent("test"); + var hostBuilder = CreateHostBuilder(agent); + using var host = await hostBuilder.StartAsync(); + var client = host.GetTestClient(); + var openaiClient = CreateOpenAIClient(client); + var openAIAgent = new OpenAIChatAgent(openaiClient, "test", "test") + .RegisterMessageConnector(); + + var response = await openAIAgent.SendAsync("Hey"); + + response.GetContent().Should().Be("Hey"); + response.Should().BeOfType(); + response.From.Should().Be("test"); + } + + [Fact] + public async Task ItReturnTextMessageWhenSendTextMessageUseStreaming() + { + var agent = new EchoAgent("test"); + var hostBuilder = CreateHostBuilder(agent); + using var host = await hostBuilder.StartAsync(); + var client = host.GetTestClient(); + var openaiClient = CreateOpenAIClient(client); + var openAIAgent = new OpenAIChatAgent(openaiClient, "test", "test") + .RegisterMessageConnector(); + + var message = new TextMessage(Role.User, "ABCDEFGHIJKLMN"); + var chunks = new List(); + await foreach (var chunk in openAIAgent.GenerateStreamingReplyAsync([message])) + { + chunk.Should().BeOfType(); + chunks.Add(chunk); + } + + var mergedChunks = string.Join("", chunks.Select(c => c.GetContent())); + mergedChunks.Should().Be("ABCDEFGHIJKLMN"); + chunks.Count.Should().Be(14); + } + + private IHostBuilder CreateHostBuilder(IAgent agent) + { + return new HostBuilder() + .ConfigureWebHost(webHost => + { + webHost.UseTestServer(); + webHost.Configure(app => + { + app.UseAgentAsOpenAIChatCompletionEndpoint(agent); + }); + }); + } + + private OpenAIClient CreateOpenAIClient(HttpClient client) + { + var clientOption = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_02_15_Preview) + { + Transport = new HttpClientTransport(client), + }; + return new OpenAIClient("api-key", clientOption); + } +} From 87bcaa43de1d14707a3830eeb332fd6c42e5dc7b Mon Sep 17 00:00:00 2001 From: Yiran Wu <32823396+yiranwu0@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:36:48 -0700 Subject: [PATCH 33/57] add stateflow to related papers (#3108) --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f14fe7a1a801..7c7ac4b85c59 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,16 @@ In addition, you can find: } ``` +[StateFlow](https://arxiv.org/abs/2403.11322) +``` +@article{wu2024stateflow, + title={StateFlow: Enhancing LLM Task-Solving through State-Driven Workflows}, + author={Wu, Yiran and Yue, Tianwei and Zhang, Shaokun and Wang, Chi and Wu, Qingyun}, + journal={arXiv preprint arXiv:2403.11322}, + year={2024} +} +``` +

โ†‘ Back to Top โ†‘ From 210e94d705964aa969cb2f40865182aa11f93f0e Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Thu, 11 Jul 2024 12:31:40 -0700 Subject: [PATCH 34/57] [.Net] Prepare release note for AutoGen.Net 0.0.16 (#3117) * add release note * update repo info --- dotnet/eng/MetaInfo.props | 2 +- dotnet/website/docfx.json | 2 ++ dotnet/website/release_note/0.0.16.md | 32 +++++++++++++++++++++ dotnet/website/release_note/toc.yml | 5 ++++ dotnet/website/{ => release_note}/update.md | 0 dotnet/website/toc.yml | 2 +- 6 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 dotnet/website/release_note/0.0.16.md create mode 100644 dotnet/website/release_note/toc.yml rename dotnet/website/{ => release_note}/update.md (100%) diff --git a/dotnet/eng/MetaInfo.props b/dotnet/eng/MetaInfo.props index 041ee0ec6c97..f43a47c8ce27 100644 --- a/dotnet/eng/MetaInfo.props +++ b/dotnet/eng/MetaInfo.props @@ -1,7 +1,7 @@ - 0.0.15 + 0.0.16 AutoGen https://microsoft.github.io/autogen-for-net/ https://github.com/microsoft/autogen diff --git a/dotnet/website/docfx.json b/dotnet/website/docfx.json index 224ef9065ca8..221cd4721e3d 100644 --- a/dotnet/website/docfx.json +++ b/dotnet/website/docfx.json @@ -32,6 +32,8 @@ "articles/**/toc.yml", "tutorial/**.md", "tutorial/**/toc.yml", + "release_note/**.md", + "release_note/**/toc.yml", "toc.yml", "*.md" ] diff --git a/dotnet/website/release_note/0.0.16.md b/dotnet/website/release_note/0.0.16.md new file mode 100644 index 000000000000..b9a190c5f793 --- /dev/null +++ b/dotnet/website/release_note/0.0.16.md @@ -0,0 +1,32 @@ +# AutoGen.Net 0.0.16 Release Notes + +We are excited to announce the release of **AutoGen.Net 0.0.16**. This release includes several new features, bug fixes, improvements, and important updates. Below are the detailed release notes: + +**[Milestone: AutoGen.Net 0.0.16](https://github.com/microsoft/autogen/milestone/4)** + +## ๐Ÿ“ฆ New Features +1. **Deprecate `IStreamingMessage`** ([#3045](https://github.com/microsoft/autogen/issues/3045)) - Replaced `IStreamingMessage` and `IStreamingMessage` with `IMessage` and `IMessage`. +2. **Add example for using ollama + LiteLLM for function call** ([#3014](https://github.com/microsoft/autogen/issues/3014)) - Added a new tutorial to the website for integrating ollama with LiteLLM for function calls. +3. **Add ReAct sample** ([#2978](https://github.com/microsoft/autogen/issues/2978)) - Added a new sample demonstrating the ReAct pattern. +4. **Support tools Anthropic Models** ([#2771](https://github.com/microsoft/autogen/issues/2771)) - Introduced support for tools like `AnthropicClient`, `AnthropicClientAgent`, and `AnthropicMessageConnector`. +5. **Propose Orchestrator for managing group chat/agentic workflow** ([#2695](https://github.com/microsoft/autogen/issues/2695)) - Introduced a customizable orchestrator interface for managing group chats and agent workflows. +6. **Run Agent as Web API** ([#2519](https://github.com/microsoft/autogen/issues/2519)) - Introduced the ability to start an OpenAI-chat-compatible web API from an arbitrary agent. + +## ๐Ÿ› Bug Fixes +1. **SourceGenerator doesn't work when function's arguments are empty** ([#2976](https://github.com/microsoft/autogen/issues/2976)) - Fixed an issue where the SourceGenerator failed when function arguments were empty. +2. **Add content field in ToolCallMessage** ([#2975](https://github.com/microsoft/autogen/issues/2975)) - Added a content property in `ToolCallMessage` to handle text content returned by the OpenAI model during tool calls. +3. **AutoGen.SourceGenerator doesnโ€™t encode `"` in structural comments** ([#2872](https://github.com/microsoft/autogen/issues/2872)) - Fixed an issue where structural comments containing `"` were not properly encoded, leading to compilation errors. + +## ๐Ÿš€ Improvements +1. **Sample update - Add getting-start samples for BasicSample project** ([#2859](https://github.com/microsoft/autogen/issues/2859)) - Re-organized the `AutoGen.BasicSample` project to include only essential getting-started examples, simplifying complex examples. +2. **Graph constructor should consider null transitions** ([#2708](https://github.com/microsoft/autogen/issues/2708)) - Updated the Graph constructor to handle cases where transitionsโ€™ values are null. + +## โš ๏ธ API-Breakchange +1. **Deprecate `IStreamingMessage`** ([#3045](https://github.com/microsoft/autogen/issues/3045)) - **Migration guide:** Deprecating `IStreamingMessage` will introduce breaking changes, particularly for `IStreamingAgent` and `IStreamingMiddleware`. Replace all `IStreamingMessage` and `IStreamingMessage` with `IMessage` and `IMessage`. + +## ๐Ÿ“š Document Update +1. **Add example for using ollama + LiteLLM for function call** ([#3014](https://github.com/microsoft/autogen/issues/3014)) - Added a tutorial to the website for using ollama with LiteLLM. + +Thank you to all the contributors for making this release possible. We encourage everyone to upgrade to AutoGen.Net 0.0.16 to take advantage of these new features and improvements. If you encounter any issues or have any feedback, please let us know. + +Happy coding! ๐Ÿš€ \ No newline at end of file diff --git a/dotnet/website/release_note/toc.yml b/dotnet/website/release_note/toc.yml new file mode 100644 index 000000000000..d3b8559a9a32 --- /dev/null +++ b/dotnet/website/release_note/toc.yml @@ -0,0 +1,5 @@ +- name: 0.0.16 + href: 0.0.16.md + +- name: 0.0.0 - 0.0.15 + href: update.md \ No newline at end of file diff --git a/dotnet/website/update.md b/dotnet/website/release_note/update.md similarity index 100% rename from dotnet/website/update.md rename to dotnet/website/release_note/update.md diff --git a/dotnet/website/toc.yml b/dotnet/website/toc.yml index b92e92d2c363..ad5d0e2b695d 100644 --- a/dotnet/website/toc.yml +++ b/dotnet/website/toc.yml @@ -8,7 +8,7 @@ href: api/ - name: Release Notes - href: update.md + href: release_note/ - name: Other Languages dropdown: true From ff311eec6c652eeb508b70765e4f69ed64308aa3 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Thu, 11 Jul 2024 16:45:05 -0700 Subject: [PATCH 35/57] fix notebook (#3093) --- ...at_group_chat_with_llamaindex_agents.ipynb | 203 ++++++++++++++++-- pyproject.toml | 1 - 2 files changed, 188 insertions(+), 16 deletions(-) diff --git a/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb index ddff0a01090e..aea134907b76 100644 --- a/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb +++ b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb @@ -26,11 +26,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install pyautogen\n", - "! pip install llama-index\n", - "! pip install llama-index-tools-wikipedia\n", - "! pip install llama-index-readers-wikipedia\n", - "! pip install wikipedia" + "%pip install pyautogen llama-index llama-index-tools-wikipedia llama-index-readers-wikipedia wikipedia" ] }, { @@ -50,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "dca301a4", "metadata": { "editable": true, @@ -65,7 +61,12 @@ "\n", "import autogen\n", "\n", - "config_list = [{\"model\": \"gpt-3.5-turbo-0125\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]" + "config_list = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\"tags\": [\"gpt-3.5-turbo\"]}, # comment out to get all\n", + ")\n", + "# When using a single openai endpoint, you can use the following:\n", + "# config_list = [{\"model\": \"gpt-3.5-turbo\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]" ] }, { @@ -78,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "2d3d298e", "metadata": {}, "outputs": [], @@ -90,7 +91,7 @@ "from llama_index.tools.wikipedia import WikipediaToolSpec\n", "\n", "llm = OpenAI(\n", - " model=\"gpt-3.5-turbo-0125\",\n", + " model=\"gpt-3.5-turbo\",\n", " temperature=0.0,\n", " api_key=os.environ.get(\"OPENAPI_API_KEY\", \"\"),\n", ")\n", @@ -125,12 +126,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "1a10c9fe-1fbc-40c6-b655-5d2256864ce8", "metadata": {}, "outputs": [], "source": [ - "from llamaindex_conversable_agent import LLamaIndexConversableAgent\n", + "from autogen.agentchat.contrib.llamaindex_conversable_agent import LLamaIndexConversableAgent\n", "\n", "llm_config = {\n", " \"temperature\": 0,\n", @@ -162,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "354b4a8f-7a96-455b-9f17-cbc19d880462", "metadata": {}, "outputs": [], @@ -179,10 +180,182 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "d5518947", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mAdmin\u001b[0m (to chat_manager):\n", + "\n", + "\n", + "What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: trip_specialist\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "> Running step 4f4f291b-5e13-495f-9871-4207e4c4bcb9. Step input: \n", + "What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.\n", + "\n", + "\u001b[1;3;38;5;200mThought: The current language of the user is: English. I need to use a tool to help me answer the question.\n", + "Action: search_data\n", + "Action Input: {'query': 'Hayao Miyazaki Tokyo'}\n", + "\u001b[0m\u001b[1;3;34mObservation: Hayao Miyazaki (ๅฎฎๅดŽ ้งฟ or ๅฎฎ๏จ‘ ้งฟ, Miyazaki Hayao, Japanese: [mija๊œœzaki hajao]; born January 5, 1941) is a Japanese animator, filmmaker, and manga artist. A founder of Studio Ghibli, he has attained international acclaim as a masterful storyteller and creator of Japanese animated feature films, and is widely regarded as one of the most accomplished filmmakers in the history of animation.\n", + "Born in Tokyo City in the Empire of Japan, Miyazaki expressed interest in manga and animation from an early age, and he joined Toei Animation in 1963. During his early years at Toei Animation he worked as an in-between artist and later collaborated with director Isao Takahata. Notable films to which Miyazaki contributed at Toei include Doggie March and Gulliver's Travels Beyond the Moon. He provided key animation to other films at Toei, such as Puss in Boots and Animal Treasure Island, before moving to A-Pro in 1971, where he co-directed Lupin the Third Part I alongside Takahata. After moving to Zuiyล Eizล (later known as Nippon Animation) in 1973, Miyazaki worked as an animator on World Masterpiece Theater, and directed the television series Future Boy Conan (1978). He joined Tokyo Movie Shinsha in 1979 to direct his first feature film The Castle of Cagliostro as well as the television series Sherlock Hound. In the same period, he began writing and illustrating the manga Nausicaรค of the Valley of the Wind (1982โ€“1994) and directed the 1984 film adaptation produced by Topcraft.\n", + "Miyazaki co-founded Studio Ghibli in 1985. He directed numerous films with Ghibli, including Laputa: Castle in the Sky (1986), My Neighbor Totoro (1988), Kiki's Delivery Service (1989), and Porco Rosso (1992). The films were met with critical and commercial success in Japan. Miyazaki's film Princess Mononoke was the first animated film ever to win the Japan Academy Film Prize for Picture of the Year, and briefly became the highest-grossing film in Japan following its release in 1997; its distribution to the Western world greatly increased Ghibli's popularity and influence outside Japan. His 2001 film Spirited Away became the highest-grossing film in Japanese history, winning the Academy Award for Best Animated Feature, and is frequently ranked among the greatest films of the 21st century. Miyazaki's later filmsโ€”Howl's Moving Castle (2004), Ponyo (2008), and The Wind Rises (2013)โ€”also enjoyed critical and commercial success. Following the release of The Wind Rises, Miyazaki announced his retirement from feature films, though he later returned to write and direct his twelfth feature film The Boy and the Heron (2023), for which he won his second Academy Award for Best Animated Feature.\n", + "Miyazaki's works are characterized by the recurrence of themes such as humanity's relationship with nature and technology, the wholesomeness of natural and traditional patterns of living, the importance of art and craftsmanship, and the difficulty of maintaining a pacifist ethic in a violent world. The protagonists of his films are often strong girls or young women, and several of his films present morally ambiguous antagonists with redeeming qualities. Miyazaki's works have been highly praised and awarded; he was named a Person of Cultural Merit for outstanding cultural contributions in November 2012, and received the Academy Honorary Award for his impact on animation and cinema in November 2014. Miyazaki has frequently been cited as an inspiration for numerous animators, directors, and writers.\n", + "\n", + "\n", + "== Early life ==\n", + "Hayao Miyazaki was born on January 5, 1941, in Tokyo City, Empire of Japan, the second of four sons. His father, Katsuji Miyazaki (born 1915), was the director of Miyazaki Airplane, his brother's company, which manufactured rudders for fighter planes during World War II. The business allowed his family to remain affluent during Miyazaki's early life. Miyazaki's father enjoyed purchasing paintings and demonstrating them to guests, but otherwise had little known artistic understanding. He said that he was in the Imperial Japanese Army around 1940; after declaring to his commanding officer that he wished not to fight because of his wife and young child, he was discharged after a lecture about disloyalty. According to Miyazaki, his father often told him about his exploits, claiming that he continued to attend nightclubs after turning 70. Katsuji Miyazaki died on March 18, 1993. After his death, Miyazaki felt that he had often looked at his father negatively and that he had never said anything \"lofty or inspiring\". He regretted not having a serious discussion with his father, and felt that he had inherited his \"anarchistic feelings and his lack of concern about embracing contradictions\".\n", + "\n", + "Miyazaki has noted that some of his earliest memories are of \"bombed-out cities\". In 1944, when he was three years old, Miyazaki's family evacuated to Utsunomiya. After the bombing of Utsunomiya in July 1945, he and his family evacuated to Kanuma. The bombing left a lasting impression on Miyazaki, then aged four. As a child, Miyazaki suffered from digestive problems, and was told that he would not live beyond 20, making him feel like an outcast. From 1947 to 1955, Miyazaki's mother Yoshiko suffered from spinal tuberculosis; she spent the first few years in hospital before being nursed from home. Yoshiko was frugal, and described as a strict, intellectual woman who regularly questioned \"socially accepted norms\". She was closest with Miyazaki, and had a strong influence on him and his later work. Yoshiko Miyazaki died in July 1983 at the age of 72.\n", + "Miyazaki began school in 1947, at an elementary school in Utsunomiya, completing the first through third grades. After his family moved back to Suginami-ku, Miyazaki completed the fourth grade at ลŒmiya Elementary School, and fifth grade at Eifuku Elementary School, which was newly established after splitting off from ลŒmiya Elementary. After graduating from Eifuku as part of the first graduating class, he attended ลŒmiya Junior High School. He aspired to become a manga artist, but discovered he could not draw people; instead, he only drew planes, tanks, and battleships for several years. Miyazaki was influenced by several manga artists, such as Tetsuji Fukushima, Soji Yamakawa and Osamu Tezuka. Miyazaki destroyed much of his early work, believing it was \"bad form\" to copy Tezuka's style as it was hindering his own development as an artist. Around this time, Miyazaki would often see movies with his father, who was an avid moviegoer; memorable films for Miyazaki include Meshi (1951) and Tasogare Sakaba (1955).\n", + "After graduating from ลŒmiya Junior High, Miyazaki attended Toyotama High School. During his third and final year, Miyazaki's interest in animation was sparked by Panda and the Magic Serpent (1958), Japan's first feature-length animated film in color; he had sneaked out to watch the film instead of studying for his entrance exams. Miyazaki later recounted that he fell in love with the film's heroine, Bai-Niang, and that the film moved him to tears and left a profound impression; he wrote that he was \"moved to the depths of [his] soul\" and that the \"pure, earnest world of the film\" affirmed a side of him that \"yearned desperately to affirm the world rather than negate it\". After graduating from Toyotama, Miyazaki attended Gakushuin University in the department of political economy, majoring in Japanese Industrial Theory. He joined the \"Children's Literature Research Club\", the \"closest thing back then to a comics club\"; he was sometimes the sole member of the club. In his free time, Miyazaki would visit his art teacher from middle school and sketch in his studio, where the two would drink and \"talk about politics, life, all sorts of things\". Around this time, he also drew manga; he never completed any stories, but accumulated thousands of pages of the beginnings of stories. He also frequently approached manga publishers to rent their stories. In 1960, Miyazaki was a bystander during the Anpo protests, having developed an interest after seeing photographs in Asahi Graph; by that point, he was too late to participate in the demonstrations. Miyazaki graduated from Gakushuin in 1963 with degrees in political science and economics.\n", + "\n", + "\n", + "== Career ==\n", + "\n", + "\n", + "=== Early career ===\n", + "\n", + "In 1963, Miyazaki was employed at Toei Animation; this was the last year the company hired regularly. After gaining employment, he began renting a four-and-a-half tatami (7.4 m2; 80 sq ft) apartment in Nerima, Tokyo; the rent was ยฅ6,000. His salary at Toei was ยฅ19,500. Miyazaki worked as an in-between artist on the theatrical feature anime Doggie March and the television anime Wolf Boy Ken (both 1963). He also worked on Gulliver's Travels Beyond the Moon (1965). He was a leader in a labor dispute soon after his arrival, and became chief secretary of Toei's labor union in 1964. Miyazaki later worked as chief animator, concept artist, and scene designer on The Great Adventure of Horus, Prince of the Sun (1968). Throughout the film's production, Miyazaki worked closely with his mentor, Yasuo ลŒtsuka, whose approach to animation profoundly influenced Miyazaki's work. Directed by Isao Takahata, with whom Miyazaki would continue to collaborate for the remainder of his career, the film was highly praised, and deemed a pivotal work in the evolution of animation. Miyazaki moved to a residence in ลŒizumigakuenchล in April 1969, after the birth of his second son.\n", + "Miyazaki provided key animation for The Wonderful World of Puss 'n Boots (1969), directed by Kimio Yabuki. He created a 12-chapter manga series as a promotional tie-in for the film; the series ran in the Sunday edition of Tokyo Shimbun from January to March 1969. Miyazaki later proposed scenes in the screenplay for Flying Phantom Ship (1969), in which military tanks would cause mass hysteria in downtown Tokyo, and was hired to storyboard and animate the scenes. Under the pseudonym Akitsu Saburล (็ง‹ๆดฅ ไธ‰ๆœ—), Miyazaki wrote and illustrated the manga People of the Desert, published in 26 installments between September 1969 and March 1970 in Boys and Girls Newspaper (ๅฐ‘ๅนดๅฐ‘ๅฅณๆ–ฐ่ž, Shลnen shลjo shinbun). He was influenced by illustrated stories such as Fukushima's Evil Lord of the Desert (ๆฒ™ๆผ ใฎ้ญ”็Ž‹, Sabaku no maล). In 1970, Miyazaki moved residence to Tokorozawa. In 1971, he developed structure, characters and designs for Hiroshi Ikeda's adaptation of Animal Treasure Island; he created the 13-part manga adaptation, printed in Tokyo Shimbun from January to March 1971. Miyazaki also provided key animation for Ali Baba and the Forty Thieves.\n", + "Miyazaki left Toei Animation in August 1971, and was hired at A-Pro, where he directed, or co-directed with Takahata, 23 episodes of Lupin the Third Part I, often using the pseudonym Teruki Tsutomu (็…งๆจน ๅ‹™). The two also began pre-production on a series based on Astrid Lindgren's Pippi Longstocking books, designing extensive storyboards; the series was canceled after Miyazaki and Takahata were unable to meet with Lindgren, and permission was refused to complete the project. In 1972 and 1973, Miyazaki wrote, designed and animated two Panda! Go, Panda! shorts, directed by Takahata. After moving from A-Pro to Zuiyล Eizล in June 1973, Miyazaki and Takahata worked on World Masterpiece Theater, which featured their animation series Heidi, Girl of the Alps, an adaptation of Johanna Spyri's Heidi. Zuiyล Eizล continued as Nippon Animation in July 1975. Miyazaki also directed the television series Future Boy Conan (1978), an adaptation of Alexander Key's The Incredible Tide.\n", + "\n", + "\n", + "=== Breakthrough films ===\n", + "Miyazaki left Nippon Animation in 1979, during the production of Anne of Green Gables; he provided scene design and organization on the first fifteen episodes. He moved to Telecom Animation Film, a subsidiary of TMS Entertainment, to direct his first feature anime film, The Castle of Cagliostro (1979), a Lupin III film. In his role at Telecom, Miyazaki helped train the second wave of employees. Miyazaki directed six episodes of Sherlock Hound in 1981, until issues with Sir Arthur Conan Doyle's estate led to a suspension in production; Miyazaki was busy with other projects by the time the issues were resolved, and the remaining episodes were directed by Kyosuke Mikuriya. They were broadcast from November 1984 to May 1985. Miyazaki also wrote the graphic novel The Journey of Shuna, inspired by the Tibetan folk tale \"Prince who became a dog\". The novel was published by Tokuma Shoten in June 1983, dramatized for radio broadcast in 1987, and published in English as Shuna's Journey in 2022. Hayao Miyazaki's Daydream Data Notes was also irregularly published from November 1984 to October 1994 in Model Graphix; selections of the stories received radio broadcast in 1995.\n", + "After the release of The Castle of Cagliostro, Miyazaki began working on his ideas for an animated film adaptation of Richard Corben's comic book Rowlf and pitched the idea to Yutaka Fujioka at TMS. In November 1980, a proposal was drawn up to acquire the film rights. Around that time, Miyazaki was also approached for a series of magazine articles by the editorial staff of Animage. During subsequent conversations, he showed his sketchbooks and discussed basic outlines for envisioned animation projects with editors Toshio Suzuki and Osamu Kameyama, who saw the potential for collaboration on their development into animation. Two projects were proposed: Warring States Demon Castle (ๆˆฆๅ›ฝ้ญ”ๅŸŽ, Sengoku ma-jล), to be set in the Sengoku period; and the adaptation of Corben's Rowlf. Both were rejected, as the company was unwilling to fund anime projects not based on existing manga, and the rights for the adaptation of Rowlf could not be secured. An agreement was reached that Miyazaki could start developing his sketches and ideas into a manga for the magazine with the proviso that it would never be made into a film. The mangaโ€”titled Nausicaรค of the Valley of the Windโ€”ran from February 1982 to March 1994. The story, as re-printed in the tankลbon volumes, spans seven volumes for a combined total of 1060 pages. Miyazaki drew the episodes primarily in pencil, and it was printed monochrome in sepia-toned ink. Miyazaki resigned from Telecom Animation Film in November 1982.\n", + "\n", + "Following the completion of Nausicaรค of the Valley of the Wind's first two volumes, Suzuki and the other editors of Animage encouraged Miyazaki to work on a film adaptation; some documentaries claim he began writing the manga after his film pitch was rejected, but Miyazaki said the manga came first. Miyazaki's imagination was sparked by the mercury poisoning of Minamata Bay and how nature responded and thrived in a poisoned environment, using it to create the film's polluted world. By this time, Miyazaki had moved to the animation studio Topcraft and was finding some of the staff to be unreliable. He eventually decided to bring on several of his previous collaborators for the film's production, including Takahata who would serve as producer. Pre-production began on May 31, 1983; Miyazaki encountered difficulties in creating the screenplay, with only sixteen chapters of the manga to work with. Takahata enlisted experimental and minimalist musician Joe Hisaishi to compose the film's score. Nausicaรค of the Valley of the Wind was released on March 11, 1984. It grossed ยฅ1.48 billion at the box office, and made an additional ยฅ742 million in distribution income. It is often seen as Miyazaki's pivotal work, cementing his reputation as an animator. It was lauded for its positive portrayal of women, particularly that of main character Nausicaรค. Several critics have labeled Nausicaรค of the Valley of the Wind as possessing anti-war and feminist themes; Miyazaki argues otherwise, stating that he only wishes to entertain. The successful cooperation on the creation of the manga and the film laid the foundation for other collaborative projects. In April 1984, Miyazaki opened his own office in Suginami Ward, naming it Nibariki.\n", + "\n", + "\n", + "=== Studio Ghibli ===\n", + "\n", + "\n", + "==== Early films (1985โ€“1996) ====\n", + "On June 15, 1985, Miyazaki and Takahata founded the animation production company Studio Ghibli as a subsidiary of Tokuma Shoten. Studio Ghibli's first film was Laputa: Castle in the Sky (1986), directed by Miyazaki. Some of the architecture in the film was also inspired by a Welsh mining town; Miyazaki witnessed the mining strike upon his first visit to Wales in 1984 and admired the miners' dedication to their work and community. Laputa was released on August 2, 1986, by the Toei Company. It sold around 775,000 tickets; Miyazaki and Suzuki expressed their disappointment with the film's box office figures. Miyazaki's following film, My Neighbor Totoro, was released alongside Takahata's Grave of the Fireflies in April 1988 to ensure Studio Ghibli's financial status. My Neighbor Totoro features the theme of the relationship between the environment and humanity, showing that harmony is the result of respecting the environment. While the film received critical acclaim, it was commercially unsuccessful at the box office. However, merchandising was successful, and the film was labeled as a cult classic.\n", + "In 1987, Studio Ghibli acquired the rights to create a film adaptation of Eiko Kadono's novel Kiki's Delivery Service. Miyazaki's work on My Neighbor Totoro prevented him from directing the adaptation; Sunao Katabuchi was chosen as director, and Nobuyuki Isshiki was hired as script writer. Miyazaki's dissatisfaction of Isshiki's first draft led him to make changes to the project, ultimately taking the role of director. Kadono was unhappy with the differences between the book and the screenplay. Miyazaki and Suzuki visited Kadono and invited her to the studio; she allowed the project to continue. The film was originally intended to be a 60-minute special, but expanded into a feature film after Miyazaki completed the storyboards and screenplay. Kiki's Delivery Service premiered on July 29, 1989. It earned ยฅ2.15 billion at the box office, and was the highest-grossing film in Japan in 1989.\n", + "From March to May 1989, Miyazaki's manga Hikลtei Jidai was published in the magazine Model Graphix. Miyazaki began production on a 45-minute in-flight film for Japan Airlines based on the manga; Suzuki ultimately extended the film into the feature-length film, titled Porco Rosso, as expectations grew. The outbreak of the Yugoslav Wars in 1991 affected Miyazaki, prompting a more sombre tone for the film; Miyazaki would later refer to the film as \"foolish\", as its mature tones were unsuitable for children. The film featured anti-war themes, which Miyazaki would later revisit. The airline remained a major investor in the film, resulting in its initial premiere as an in-flight film, prior to its theatrical release on July 18, 1992. The film was commercially successful and remained one of the highest-grossing films in Japan for several years.\n", + "Studio Ghibli set up its headquarters in Koganei, Tokyo in August 1992. In November 1992, two television spots directed by Miyazaki were broadcast by Nippon Television Network (NTV): Sora Iro no Tane, a 90-second spot adapted from the illustrated story Sora Iro no Tane by Rieko Nakagawa and Yuriko Omura; and Nandarou, a series of five advertisements featuring an undefinable creature. Miyazaki designed the storyboards and wrote the screenplay for Whisper of the Heart (1995), directed by Yoshifumi Kondล.\n", + "\n", + "\n", + "==== Global emergence (1997โ€“2008) ====\n", + "Miyazaki began work on the initial storyboards for Princess Mononoke in August 1994, based on preliminary thoughts and sketches from the late 1970s. While experiencing writer's block during production, Miyazaki accepted a request for the creation of On Your Mark, a music video for the song of the same name by Chage and Aska. In the production of the video, Miyazaki experimented with computer animation to supplement traditional animation. On Your Mark premiered as a short before Whisper of the Heart. Despite the video's popularity, Suzuki said that it was not given \"100 percent\" focus.\n", + "\n", + "In May 1995, Miyazaki took a group of artists and animators to the ancient forests of Yakushima and the mountains of Shirakami-Sanchi, taking photographs and making sketches. The landscapes in the film were inspired by Yakushima. In Princess Mononoke, Miyazaki revisited the ecological and political themes of Nausicaรค of the Valley of the Wind. Miyazaki supervised the 144,000 cels in the film, about 80,000 of which were key animation. Princess Mononoke was produced with an estimated budget of ยฅ2.35 billion (approximately US$23.5 million), making it the most expensive Japanese animated film at the time. Approximately fifteen minutes of the film uses computer animation: about five minutes uses techniques such as 3D rendering, digital composition, and texture mapping; the remaining ten minutes uses digital ink and paint. While the original intention was to digitally paint 5,000 of the film's frames, time constraints doubled this, though it remained below ten percent of the final film.\n", + "Upon its premiere on July 12, 1997, Princess Mononoke was critically acclaimed, becoming the first animated film to win the Japan Academy Film Prize for Picture of the Year. The film was also commercially successful, becoming the highest-grossing film in Japan for several months. Miramax Films purchased the film's distributions rights for North America; while it was largely unsuccessful at the box office, grossing about US$2.3 million, it was seen as the introduction of Studio Ghibli to global markets. Miyazaki claimed Princess Mononoke would be his final film. Tokuma Shoten merged with Studio Ghibli in June 1997. Miyazaki left Studio Ghibli on January 14, 1998, to create a new studio called Butaya, to be succeeded by Kondล; however, Kondล's death impacted Miyazaki, and he returned to Studio Ghibli on January 16, 1999.\n", + "Miyazaki's next film was conceived while on vacation at a mountain cabin with his family and five young girls who were family friends. Miyazaki realized that he had not created a film for 10-year-old girls, and set out to do so. He read shลjล manga magazines like Nakayoshi and Ribon for inspiration, but felt they only offered subjects on \"crushes and romance\", which is not what the girls \"held dear in their hearts\". He decided to produce the film about a female heroine whom they could look up to. Production of the film, titled Spirited Away, commenced in 2000 on a budget of ยฅ1.9 billion (US$15 million). As with Princess Mononoke, the staff experimented with computer animation, but kept the technology at a level to enhance the story, not to \"steal the show\". Spirited Away deals with symbols of human greed, symbolizing the 1980s Japanese asset price bubble, and a liminal journey through the realm of spirits. The film was released on July 20, 2001; it received critical acclaim, and is considered among the greatest films of the 2000s. It won the Japan Academy Film Prize for Picture of the Year, and the Academy Award for Best Animated Feature. The film was also commercially successful, earning ยฅ30.4 billion (US$289.1 million) at the box office. It became the highest-grossing film in Japan, a record it maintained for almost 20 years. Following the death of Tokuma in September 2000, Miyazaki served as the head of his funeral committee.\n", + "In September 2001, Studio Ghibli announced the production of Howl's Moving Castle, based on the novel by Diana Wynne Jones. Mamoru Hosoda of Toei Animation was originally selected to direct the film, but disagreements between Hosoda and Studio Ghibli executives led to the project's abandonment. After six months, Studio Ghibli resurrected the project. Miyazaki was inspired to direct the film upon reading Jones' novel, and was struck by the image of a castle moving around the countryside; the novel does not explain how the castle moved, which led to Miyazaki's designs. He traveled to Colmar and Riquewihr in Alsace, France, to study the architecture and the surroundings for the film's setting. Additional inspiration came from the concepts of future technology in Albert Robida's work. It was released on November 20, 2004, and received widespread critical acclaim. The film received the Osella Award for Technical Excellence at the 61st Venice International Film Festival, and was nominated for the Academy Award for Best Animated Feature. In Japan, the film grossed a record $14.5 million in its first week of release. It remains among the highest-grossing films in Japan, with a worldwide gross of over ยฅ19.3 billion. Miyazaki received the honorary Golden Lion for Lifetime Achievement award at the 62nd Venice International Film Festival in 2005.\n", + "In March 2005, Studio Ghibli split from Tokuma Shoten. In the 1980s, Miyazaki had contacted Ursula K. Le Guin expressing interest in producing an adaptation of her Earthsea novels; unaware of Miyazaki's work, Le Guin declined. Upon watching My Neighbor Totoro several years later, Le Guin expressed approval to the concept of the adaptation. She met with Suzuki in August 2005, who wanted Miyazaki's son Goro to direct the film, as Miyazaki had wished to retire. Disappointed that Miyazaki was not directing, but under the impression that he would supervise his son's work, Le Guin approved of the film's production. Miyazaki later publicly opposed and criticized Gorล's appointment as director. Upon Miyazaki's viewing of the film, he wrote a message for his son: \"It was made honestly, so it was good\".\n", + "Miyazaki designed the covers for several manga novels in 2006, including A Trip to Tynemouth; he also worked as editor, and created a short manga for the book. Miyazaki's next film, Ponyo, began production in May 2006. It was initially inspired by \"The Little Mermaid\" by Hans Christian Andersen, though began to take its own form as production continued. Miyazaki aimed for the film to celebrate the innocence and cheerfulness of a child's universe. He intended for it to only use traditional animation, and was intimately involved with the artwork. He preferred to draw the sea and waves himself, as he enjoyed experimenting. Ponyo features 170,000 framesโ€”a record for Miyazaki. The film's seaside village was inspired by Tomonoura, a town in Setonaikai National Park, where Miyazaki stayed in 2005. The main character, Sลsuke, is based on Gorล. Following its release on July 19, 2008, Ponyo was critically acclaimed, receiving Animation of the Year at the 32nd Japan Academy Film Prize. The film was also a commercial success, earning ยฅ10 billion (US$93.2 million) in its first month and ยฅ15.5 billion by the end of 2008, placing it among the highest-grossing films in Japan.\n", + "\n", + "\n", + "==== Later films (2009โ€“present) ====\n", + "\n", + "In early 2009, Miyazaki began writing a manga called Kaze Tachinu (้ขจ็ซ‹ใกใฌ, The Wind Rises), telling the story of Mitsubishi A6M Zero fighter designer Jiro Horikoshi. The manga was first published in two issues of the Model Graphix magazine, published on February 25 and March 25, 2009. Miyazaki later co-wrote the screenplay for Arrietty (2010) and From Up on Poppy Hill (2011), directed by Hiromasa Yonebayashi and Gorล Miyazaki respectively. Miyazaki wanted his next film to be a sequel to Ponyo, but Suzuki convinced him to instead adapt Kaze Tachinu to film. In November 2012, Studio Ghibli announced the production of The Wind Rises, based on Kaze Tachinu, to be released alongside Takahata's The Tale of the Princess Kaguya.\n", + "Miyazaki was inspired to create The Wind Rises after reading a quote from Horikoshi: \"All I wanted to do was to make something beautiful\". Several scenes in The Wind Rises were inspired by Tatsuo Hori's novel The Wind Has Risen (้ขจ็ซ‹ใกใฌ), in which Hori wrote about his life experiences with his fiancรฉe before she died from tuberculosis. The female lead character's name, Naoko Satomi, was borrowed from Hori's novel Naoko (่œ็ฉ‚ๅญ). The Wind Rises continues to reflect Miyazaki's pacifist stance, continuing the themes of his earlier works, despite stating that condemning war was not the intention of the film. The film premiered on July 20, 2013, and received critical acclaim; it was named Animation of the Year at the 37th Japan Academy Film Prize, and was nominated for Best Animated Feature at the 86th Academy Awards. It was also commercially successful, grossing ยฅ11.6 billion (US$110 million) at the Japanese box office, becoming the highest-grossing film in Japan in 2013.\n", + "In September 2013, Miyazaki announced that he was retiring from the production of feature films due to his age, but wished to continue working on the displays at the Studio Ghibli Museum. Miyazaki was awarded the Academy Honorary Award at the Governors Awards in November 2014. He developed Boro the Caterpillar, an animated short film which was first discussed during pre-production for Princess Mononoke. It was screened exclusively at the Studio Ghibli Museum in July 2017. Around this time, Miyazaki was working on a manga titled Teppo Samurai. In February 2019, a four-part documentary was broadcast on the NHK network titled 10 Years with Hayao Miyazaki, documenting production of his films in his private studio. In 2019, Miyazaki approved a musical adaptation of Nausicaรค of the Valley of the Wind, as it was performed by a kabuki troupe.\n", + "In August 2016, Miyazaki proposed a new feature-length film, Kimi-tachi wa Dล Ikiru ka (titled The Boy and the Heron in English), on which he began animation work without receiving official approval. The film opened in Japanese theaters on July 14, 2023. It was preceded by a minimal marketing campaign, forgoing trailers, commercials, and advertisements, a response from Suzuki to his perceived oversaturation of marketing materials in mainstream films. Despite claims that The Boy and the Heron would be Miyazaki's final film, Studio Ghibli vice president Junichi Nishioka said in September 2023 that Miyazaki continued to attend the office daily to plan his next film. Suzuki said he could no longer convince Miyazaki to retire. The Boy and the Heron won Miyazaki his second Academy Award for Best Animated Feature at the 96th Academy Awards, becoming the oldest director to do so; Miyazaki did not attend the show due to his advanced age.\n", + "\n", + "\n", + "== Views ==\n", + "\n", + "Miyazaki has often criticized the state of the animation industry, stating that some animators lack a foundational understanding of their subjects and do not prioritize realism. He is particularly critical of Japanese animation, saying that anime is \"produced by humans who can't stand looking at other humans ... that's why the industry is full of otakuโ€Š!\". He has frequently criticized otaku, including \"fanatics\" of guns and fighter aircraft, declaring it a \"fetish\" and refusing to identify himself as such. He bemoaned the state of Disney animated films in 1988, saying \"they show nothing but contempt for the audience\".\n", + "In 2013, Miyazaki criticized Japanese Prime Minister Shinzo Abe's policies and the proposed Constitutional amendment that would allow Abe to revise the clause outlawing war as a means to settle international disputes. Miyazaki felt Abe wished to \"leave his name in history as a great man who revised the Constitution and its interpretation\", describing it as \"despicable\" and stating \"People who don't think enough shouldn't meddle with the constitution\". In 2015, Miyazaki disapproved Abe's denial of Japan's military aggression, stating Japan \"should clearly say that [they] inflicted enormous damage on China and express deep remorse over it\". He felt the government should give a \"proper apology\" to Korean comfort women who were forced to service the Japanese army during World War II and suggested the Senkaku Islands be \"split in half\" or controlled by both Japan and China. After the release of The Wind Rises in 2013, some online critics labeled Miyazaki a \"traitor\" and \"anti-Japanese\", describing the film as overly \"left-wing\"; Miyazaki recognized leftist values in his movies, citing his influence by and appreciation of communism as defined by Karl Marx, but criticized the Soviet Union's political system.\n", + "In 2003, Miyazaki refused to attend the 75th Academy Awards in Hollywood in protest of the United States's involvement in the Iraq War, and later said he \"didn't want to visit a country that was bombing Iraq\". He did not publicly express this opinion at the request of his producer until 2009 when he lifted his boycott and attended San Diego Comic Con International as a favor to his friend John Lasseter. Miyazaki also expressed his opinion about the terrorist attack at the offices of the French satirical magazine Charlie Hebdo, criticizing the magazine's decision to publish the content cited as the catalyst for the incident; he felt caricatures should be made of politicians, not cultures. In November 2016, Miyazaki stated that he believed \"many of the people who voted for Brexit and Trump\" were affected by the increase in unemployment due to companies \"building cars in Mexico because of low wages and [selling] them in the US\". He did not think that Donald Trump would be elected president, calling it \"a terrible thing\", and said that Trump's political opponent Hillary Clinton was \"terrible as well\".\n", + "\n", + "\n", + "== Themes ==\n", + "Miyazaki's works are characterized by the recurrence of themes such as feminism, environmentalism, pacifism, love, and family. His narratives are also notable for not pitting a hero against an unsympathetic antagonist; Miyazaki felt Spirited Away's Chihiro \"manages not because she has destroyed the 'evil', but because she has acquired the ability to survive\".\n", + "Miyazaki's films often emphasize environmentalism and the Earth's fragility. Margaret Talbot stated that Miyazaki dislikes modern technology, and believes much of modern culture is \"thin and shallow and fake\"; he anticipates a time with \"no more high-rises\". Miyazaki felt frustrated growing up in the Shลwa period from 1955 to 1965 because \"natureโ€”the mountains and riversโ€”was being destroyed in the name of economic progress\". Peter Schellhase of The Imaginative Conservative identified that several antagonists of Miyazaki's films \"attempt to dominate nature in pursuit of political domination, and are ultimately destructive to both nature and human civilization\". Miyazaki is critical of exploitation under both communism and capitalism, as well as globalization and its effects on modern life, believing that \"a company is common property of the people that work there\". Ram Prakash Dwivedi identified values of Mahatma Gandhi in the films of Miyazaki.\n", + "Several of Miyazaki's films feature anti-war themes. Daisuke Akimoto of Animation Studies categorized Porco Rosso as \"anti-war propaganda\" and felt the protagonist, Porco, transforms into a pig partly due to his extreme distaste of militarism. Akimoto also argues that The Wind Rises reflects Miyazaki's \"antiwar pacifism\", despite the latter stating that the film does not attempt to \"denounce\" war. Schellhase also identifies Princess Mononoke as a pacifist film due to the protagonist, Ashitaka; instead of joining the campaign of revenge against humankind, as his ethnic history would lead him to do, Ashitaka strives for peace. David Loy and Linda Goodhew argue that both Nausicaรค of the Valley of the Wind and Princess Mononoke do not depict traditional evil, but the Buddhist roots of evil: greed, ill will, and delusion; according to Buddhism, the roots of evil must transform into \"generosity, loving-kindness and wisdom\" in order to overcome suffering, and both Nausicaรค and Ashitaka accomplish this. When characters in Miyazaki's films are forced to engage in violence, it is shown as being a difficult task; in Howl's Moving Castle, Howl is forced to fight an inescapable battle in defense of those he loves, and it almost destroys him, though he is ultimately saved by Sophie's love and bravery.\n", + "Suzuki described Miyazaki as a feminist in reference to his attitude to female workers. Miyazaki has described his female characters as \"brave, self-sufficient girls that don't think twice about fighting for what they believe in with all their heart\", stating that they may \"need a friend, or a supporter, but never a saviour\" and that \"any woman is just as capable of being a hero as any man\". Nausicaรค of the Valley of the Wind was lauded for its positive portrayal of women, particularly protagonist Nausicaรค. Schellhase noted that the female characters in Miyazaki's films are not objectified or sexualized, and possess complex and individual characteristics absent from Hollywood productions. Schellhase also identified a \"coming of age\" element for the heroines in Miyazaki's films, as they each discover \"individual personality and strengths\". Gabrielle Bellot of The Atlantic wrote that, in his films, Miyazaki \"shows a keen understanding of the complexities of what it might mean to be a woman\". In particular, Bellot cites Nausicaรค of the Valley of the Wind, praising the film's challenging of gender expectations, and the strong and independent nature of Nausicaรค. Bellot also noted that Princess Mononoke's San represents the \"conflict between selfhood and expression\".\n", + "Miyazaki is concerned with the sense of wonder in young people, seeking to maintain themes of love and family in his films. Michael Toscano of Curator found that Miyazaki \"fears Japanese children are dimmed by a culture of overconsumption, overprotection, utilitarian education, careerism, techno-industrialism, and a secularism that is swallowing Japan's native animism\". Schellhase wrote that several of Miyazaki's works feature themes of love and romance, but felt emphasis is placed on \"the way lonely and vulnerable individuals are integrated into relationships of mutual reliance and responsibility, which generally benefit everyone around them\". He also found that many of the protagonists in Miyazaki's films present an idealized image of families, whereas others are dysfunctional.\n", + "\n", + "\n", + "== Creation process and influences ==\n", + "Miyazaki forgoes traditional screenplays in his productions, instead developing the film's narrative as he designs the storyboards. \"We never know where the story will go but we just keep working on the film as it develops,\" he said. In each of his films, Miyazaki has employed traditional animation methods, drawing each frame by hand; computer-generated imagery has been employed in several of his later films, beginning with Princess Mononoke, to \"enrich the visual look\", though he ensures that each film can \"retain the right ratio between working by hand and computer ... and still be able to call my films 2D\". He oversees every frame of his films. For character designs, Miyazaki draws original drafts used by animation directors to create reference sheets, which are then corrected by Miyazaki in his style.\n", + "Miyazaki has cited several Japanese artists as his influences, including Sanpei Shirato, Osamu Tezuka, Soji Yamakawa, and Isao Takahata. A number of Western authors have also influenced his works, including Frรฉdรฉric Back, Lewis Carroll, Roald Dahl, Jean Giraud, Paul Grimault, Ursula K. Le Guin, and Yuri Norstein, as well as animation studio Aardman Animations (specifically the works of Nick Park). Specific works that have influenced Miyazaki include Animal Farm (1945), The Snow Queen (1957), and The King and the Mockingbird (1980); The Snow Queen is said to be the true catalyst for Miyazaki's filmography, influencing his training and work. When animating young children, Miyazaki often takes inspiration from his friends' children, as well as memories of his own childhood.\n", + "\n", + "\n", + "== Personal life ==\n", + "\n", + "Miyazaki married fellow animator Akemi ลŒta in October 1965; the two had met while colleagues at Toei Animation. The couple have two sons: Goro, born in January 1967, and Keisuke, born in April 1969. Miyazaki felt that becoming a father changed him, as he tried to produce work that would please his children. Miyazaki initially fulfilled a promise to his wife that they would both continue to work after Goro's birth, dropping him off at preschool for the day; however, upon seeing Goro's exhaustion walking home one day, Miyazaki decided that they could not continue, and his wife stayed at home to raise their children. Miyazaki's dedication to his work harmed his relationship with his children, as he was often absent. Goro watched his father's works in an attempt to \"understand\" him, since the two rarely talked. Miyazaki said that he \"tried to be a good father, but in the end [he] wasn't a very good parent\". During the production of Tales from Earthsea in 2006, Goro said that his father \"gets zero marks as a father but full marks as a director of animated films\".\n", + "Goro worked at a landscape design firm before beginning to work at the Ghibli Museum; he designed the garden on its rooftop and eventually became its curator. Keisuke studied forestry at Shinshu University and works as a wood artist; he designed a woodcut print that appears in Whisper of the Heart. Miyazaki's niece, Mei Okuyama, who was the inspiration behind the character Mei in My Neighbor Totoro, is married to animation artist Daisuke Tsutsumi.\n", + "\n", + "\n", + "== Legacy ==\n", + "Miyazaki was described as the \"godfather of animation in Japan\" by BBC's Tessa Wong in 2016, citing his craftsmanship and humanity, the themes of his films, and his inspiration to younger artists. Courtney Lanning of Arkansas Democrat-Gazette named him one of the world's greatest animators, comparing him to Osamu Tezuka and Walt Disney. Swapnil Dhruv Bose of Far Out Magazine wrote that Miyazaki's work \"has shaped not only the future of animation but also filmmaking in general\", and that it helped \"generation after generation of young viewers to observe the magic that exists in the mundane\". Richard James Havis of South China Morning Post called him a \"genius ... who sets exacting standards for himself, his peers and studio staff\". Paste's Toussaint Egan described Miyazaki as \"one of anime's great auteurs\", whose \"stories of such singular thematic vision and unmistakable aesthetic\" captured viewers otherwise unfamiliar with anime. Miyazaki became the subject of an exhibit at the Academy Museum of Motion Pictures in Los Angeles in 2021, featuring over 400 objects from his films.\n", + "Miyazaki has frequently been cited as an inspiration to numerous animators, directors and writers around the world, including Wes Anderson, James Cameron, Dean DeBlois, Guillermo del Toro, Pete Docter, Mamoru Hosoda, Bong Joon-ho, Travis Knight, John Lasseter, Nick Park, Henry Selick, Makoto Shinkai, and Steven Spielberg. Glen Keane said Miyazaki is a \"huge influence\" on Walt Disney Animation Studios and has been \"part of our heritage\" ever since The Rescuers Down Under (1990). The Disney Renaissance era was also prompted by competition with the development of Miyazaki's films. Artists from Pixar and Aardman Studios signed a tribute stating, \"You're our inspiration, Miyazaki-san!\" He has also been cited as inspiration for video game designers including Shigeru Miyamoto on The Legend of Zelda and Hironobu Sakaguchi on Final Fantasy, as well as the television series Avatar: The Last Airbender, and the video game Ori and the Blind Forest (2015).\n", + "Studio Ghibli has searched for some time for Miyazaki and Suzuki's successor to lead the studio; Kondล, the director of Whisper of the Heart, was initially considered, but died from a sudden heart attack in 1998. Some candidates were considered by 2023โ€”including Miyazaki's son Goro, who declinedโ€”but the studio was not able to find a successor.\n", + "\n", + "\n", + "== Selected filmography ==\n", + "\n", + "The Castle of Cagliostro (1979)\n", + "Nausicaรค of the Valley of the Wind (1984)\n", + "Laputa: Castle in the Sky (1986)\n", + "My Neighbor Totoro (1988)\n", + "Kiki's Delivery Service (1989)\n", + "Porco Rosso (1992)\n", + "Princess Mononoke (1997)\n", + "Spirited Away (2001)\n", + "Howl's Moving Castle (2004)\n", + "Ponyo (2008)\n", + "The Wind Rises (2013)\n", + "The Boy and the Heron (2023)\n", + "\n", + "\n", + "== Awards and nominations ==\n", + "\n", + "Miyazaki won the ลŒfuji Noburล Award at the Mainichi Film Awards for The Castle of Cagliostro (1979), Nausicaรค of the Valley of the Wind (1984), Laputa: Castle in the Sky (1986), and My Neighbor Totoro (1988), and the Mainichi Film Award for Best Animation Film for Kiki's Delivery Service (1989), Porco Rosso (1992), Princess Mononoke (1997), Spirited Away (2001), and Whale Hunt (2001). Spirited Away and The Boy and the Heron were awarded the Academy Award for Best Animated Feature, while Howl's Moving Castle (2004) and The Wind Rises (2013) received nominations. He was named a Person of Cultural Merit by the Japanese government in November 2012, for outstanding cultural contributions. In 2024, Time named him one of the 100 most influential people in the world, and Gold House honored him on its Most Impactful Asians A100 list. His other accolades include several Annie Awards, Japan Academy Film Prizes, Kinema Junpo Awards, and Tokyo Anime Awards.\n", + "\n", + "\n", + "== Notes ==\n", + "\n", + "\n", + "== References ==\n", + "\n", + "\n", + "== Sources ==\n", + "\n", + "\n", + "== External links ==\n", + "\n", + "Studio Ghibli (in Japanese)\n", + "Hayao Miyazaki at Anime News Network's encyclopedia\n", + "Hayao Miyazaki at IMDb\n", + "Hayao Miyazaki at Library of Congress, with 14 library catalogue records\n", + "\u001b[0m> Running step 015c82f1-c7ff-4de0-9930-ab2467424c1c. Step input: None\n", + "\u001b[1;3;38;5;200mThought: I have found detailed information about Hayao Miyazaki, his works, and his life in Tokyo. Now, I can provide a summary of the relevant details related to Tokyo and his movies like \"Spirited Away.\"\n", + "Answer: Hayao Miyazaki, a renowned Japanese animator, filmmaker, and manga artist, was born in Tokyo City, Japan. He expressed interest in manga and animation from a young age and joined Toei Animation in 1963. Miyazaki has contributed to various notable films and series, including \"The Castle of Cagliostro,\" \"Princess Mononoke,\" \"Spirited Away,\" and more. He co-founded Studio Ghibli in 1985, where he directed several successful films like \"My Neighbor Totoro,\" \"Kiki's Delivery Service,\" and \"Porco Rosso.\" His films often explore themes such as humanity's relationship with nature, the importance of art and craftsmanship, and the challenges of maintaining pacifism in a violent world.\n", + "\n", + "One of Miyazaki's most famous works, \"Spirited Away,\" became the highest-grossing film in Japanese history and won the Academy Award for Best Animated Feature. The film is known for its rich storytelling and captivating animation. Miyazaki's works are characterized by strong female protagonists, morally ambiguous antagonists, and themes of environmentalism, love, and family. His films have received critical acclaim and commercial success, influencing a generation of animators and filmmakers worldwide.\n", + "\n", + "Miyazaki's legacy extends beyond his films, as he has been recognized for his craftsmanship, thematic depth, and influence on the animation industry. His dedication to hand-drawn animation and storytelling has inspired numerous artists and filmmakers globally.\n", + "\u001b[0m\u001b[33mtrip_specialist\u001b[0m (to chat_manager):\n", + "\n", + "Hayao Miyazaki, a renowned Japanese animator, filmmaker, and manga artist, was born in Tokyo City, Japan. He expressed interest in manga and animation from a young age and joined Toei Animation in 1963. Miyazaki has contributed to various notable films and series, including \"The Castle of Cagliostro,\" \"Princess Mononoke,\" \"Spirited Away,\" and more. He co-founded Studio Ghibli in 1985, where he directed several successful films like \"My Neighbor Totoro,\" \"Kiki's Delivery Service,\" and \"Porco Rosso.\" His films often explore themes such as humanity's relationship with nature, the importance of art and craftsmanship, and the challenges of maintaining pacifism in a violent world.\n", + "\n", + "One of Miyazaki's most famous works, \"Spirited Away,\" became the highest-grossing film in Japanese history and won the Academy Award for Best Animated Feature. The film is known for its rich storytelling and captivating animation. Miyazaki's works are characterized by strong female protagonists, morally ambiguous antagonists, and themes of environmentalism, love, and family. His films have received critical acclaim and commercial success, influencing a generation of animators and filmmakers worldwide.\n", + "\n", + "Miyazaki's legacy extends beyond his films, as he has been recognized for his craftsmanship, thematic depth, and influence on the animation industry. His dedication to hand-drawn animation and storytelling has inspired numerous artists and filmmakers globally.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Admin\n", + "\u001b[0m\n" + ] + } + ], "source": [ "chat_result = user_proxy.initiate_chat(\n", " manager,\n", @@ -217,7 +390,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/pyproject.toml b/pyproject.toml index 7981ef4b43d5..107c438a7f41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ exclude = [ "math_utils\\.py$", "**/cap/py/autogencap/proto/*", ] -ignore-init-module-imports = true unfixable = ["F401"] [tool.ruff.lint.mccabe] From 24d509c1b61c9df7ddc5dab095488636a1284441 Mon Sep 17 00:00:00 2001 From: Mark Ward <90335263+MarkWard0110@users.noreply.github.com> Date: Sat, 13 Jul 2024 15:20:56 -0500 Subject: [PATCH 36/57] middleware examples updated to return modified message passing assertion. modified the default agent reply so that it is different from the user's prompt (#3128) --- .../CodeSnippet/MiddlewareAgentCodeSnippet.cs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs index 320afd0de679..795606ebc85c 100644 --- a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs +++ b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs @@ -13,38 +13,48 @@ public class MiddlewareAgentCodeSnippet public async Task CreateMiddlewareAgentAsync() { #region create_middleware_agent_with_original_agent - // Create an agent that always replies "Hello World" - IAgent agent = new DefaultReplyAgent(name: "assistant", defaultReply: "Hello World"); + // Create an agent that always replies "Hi!" + IAgent agent = new DefaultReplyAgent(name: "assistant", defaultReply: "Hi!"); // Create a middleware agent on top of default reply agent var middlewareAgent = new MiddlewareAgent(innerAgent: agent); middlewareAgent.Use(async (messages, options, agent, ct) => { var lastMessage = messages.Last() as TextMessage; - lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + if (lastMessage != null && lastMessage.Content.Contains("Hello World")) + { + lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + return lastMessage; + } + return await agent.GenerateReplyAsync(messages, options, ct); }); var reply = await middlewareAgent.SendAsync("Hello World"); reply.GetContent().Should().Be("[middleware 0] Hello World"); + reply = await middlewareAgent.SendAsync("Hello AI!"); + reply.GetContent().Should().Be("Hi!"); #endregion create_middleware_agent_with_original_agent #region register_middleware_agent middlewareAgent = agent.RegisterMiddleware(async (messages, options, agent, ct) => { var lastMessage = messages.Last() as TextMessage; - lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + if (lastMessage != null && lastMessage.Content.Contains("Hello World")) + { + lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + return lastMessage; + } + return await agent.GenerateReplyAsync(messages, options, ct); }); #endregion register_middleware_agent #region short_circuit_middleware_agent - // This middleware will short circuit the agent and return the last message directly. + // This middleware will short circuit the agent and return a message directly. middlewareAgent.Use(async (messages, options, agent, ct) => { - var lastMessage = messages.Last() as TextMessage; - lastMessage.Content = $"[middleware shortcut]"; - return lastMessage; + return new TextMessage(Role.Assistant, $"[middleware shortcut]"); }); #endregion short_circuit_middleware_agent } From 5b1dc3bf63b26ea4c9598901dc0095e345ce79b4 Mon Sep 17 00:00:00 2001 From: Anush Date: Mon, 15 Jul 2024 18:49:58 +0530 Subject: [PATCH 37/57] feat: Qdrant support for the VectorDB interface (#3035) * feat: Qdrant support * chore: pre-defined vector db * Fix issues --------- Co-authored-by: Li Jiang --- .../contrib/retrieve_user_proxy_agent.py | 7 +- autogen/agentchat/contrib/vectordb/base.py | 6 +- autogen/agentchat/contrib/vectordb/qdrant.py | 322 ++++++++++++++++++ .../agentchat/contrib/vectordb/test_qdrant.py | 68 ++++ 4 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 autogen/agentchat/contrib/vectordb/qdrant.py create mode 100644 test/agentchat/contrib/vectordb/test_qdrant.py diff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py index 59a4abccb1d6..1471afb07eaf 100644 --- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py +++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py @@ -1,6 +1,7 @@ import hashlib import os import re +import uuid from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union from IPython import get_ipython @@ -365,7 +366,11 @@ def _init_db(self): else: all_docs_ids = set() - chunk_ids = [hashlib.blake2b(chunk.encode("utf-8")).hexdigest()[:HASH_LENGTH] for chunk in chunks] + chunk_ids = ( + [hashlib.blake2b(chunk.encode("utf-8")).hexdigest()[:HASH_LENGTH] for chunk in chunks] + if not self._vector_db.type == "qdrant" + else [str(uuid.UUID(hex=hashlib.md5(chunk.encode("utf-8")).hexdigest())) for chunk in chunks] + ) chunk_ids_set = set(chunk_ids) chunk_ids_set_idx = [chunk_ids.index(hash_value) for hash_value in chunk_ids_set] docs = [ diff --git a/autogen/agentchat/contrib/vectordb/base.py b/autogen/agentchat/contrib/vectordb/base.py index 29a080086193..1531a6494cef 100644 --- a/autogen/agentchat/contrib/vectordb/base.py +++ b/autogen/agentchat/contrib/vectordb/base.py @@ -185,7 +185,7 @@ class VectorDBFactory: Factory class for creating vector databases. """ - PREDEFINED_VECTOR_DB = ["chroma", "pgvector"] + PREDEFINED_VECTOR_DB = ["chroma", "pgvector", "qdrant"] @staticmethod def create_vector_db(db_type: str, **kwargs) -> VectorDB: @@ -207,6 +207,10 @@ def create_vector_db(db_type: str, **kwargs) -> VectorDB: from .pgvectordb import PGVectorDB return PGVectorDB(**kwargs) + if db_type.lower() in ["qdrant", "qdrantdb"]: + from .qdrant import QdrantVectorDB + + return QdrantVectorDB(**kwargs) else: raise ValueError( f"Unsupported vector database type: {db_type}. Valid types are {VectorDBFactory.PREDEFINED_VECTOR_DB}." diff --git a/autogen/agentchat/contrib/vectordb/qdrant.py b/autogen/agentchat/contrib/vectordb/qdrant.py new file mode 100644 index 000000000000..479d7a485270 --- /dev/null +++ b/autogen/agentchat/contrib/vectordb/qdrant.py @@ -0,0 +1,322 @@ +import abc +import logging +import os +from typing import Callable, List, Optional, Sequence, Tuple, Union + +from .base import Document, ItemID, QueryResults, VectorDB +from .utils import get_logger + +try: + from qdrant_client import QdrantClient, models +except ImportError: + raise ImportError("Please install qdrant-client: `pip install qdrant-client`") + +logger = get_logger(__name__) + +Embeddings = Union[Sequence[float], Sequence[int]] + + +class EmbeddingFunction(abc.ABC): + @abc.abstractmethod + def __call__(self, inputs: List[str]) -> List[Embeddings]: + raise NotImplementedError + + +class FastEmbedEmbeddingFunction(EmbeddingFunction): + """Embedding function implementation using FastEmbed - https://qdrant.github.io/fastembed.""" + + def __init__( + self, + model_name: str = "BAAI/bge-small-en-v1.5", + batch_size: int = 256, + cache_dir: Optional[str] = None, + threads: Optional[int] = None, + parallel: Optional[int] = None, + **kwargs, + ): + """Initialize fastembed.TextEmbedding. + + Args: + model_name (str): The name of the model to use. Defaults to `"BAAI/bge-small-en-v1.5"`. + batch_size (int): Batch size for encoding. Higher values will use more memory, but be faster.\ + Defaults to 256. + cache_dir (str, optional): The path to the model cache directory.\ + Can also be set using the `FASTEMBED_CACHE_PATH` env variable. + threads (int, optional): The number of threads single onnxruntime session can use. + parallel (int, optional): If `>1`, data-parallel encoding will be used, recommended for large datasets.\ + If `0`, use all available cores.\ + If `None`, don't use data-parallel processing, use default onnxruntime threading.\ + Defaults to None. + **kwargs: Additional options to pass to fastembed.TextEmbedding + Raises: + ValueError: If the model_name is not in the format / e.g. BAAI/bge-small-en-v1.5. + """ + try: + from fastembed import TextEmbedding + except ImportError as e: + raise ValueError( + "The 'fastembed' package is not installed. Please install it with `pip install fastembed`", + ) from e + self._batch_size = batch_size + self._parallel = parallel + self._model = TextEmbedding(model_name=model_name, cache_dir=cache_dir, threads=threads, **kwargs) + + def __call__(self, inputs: List[str]) -> List[Embeddings]: + embeddings = self._model.embed(inputs, batch_size=self._batch_size, parallel=self._parallel) + + return [embedding.tolist() for embedding in embeddings] + + +class QdrantVectorDB(VectorDB): + """ + A vector database implementation that uses Qdrant as the backend. + """ + + def __init__( + self, + *, + client=None, + embedding_function: EmbeddingFunction = None, + content_payload_key: str = "_content", + metadata_payload_key: str = "_metadata", + collection_options: dict = {}, + **kwargs, + ) -> None: + """ + Initialize the vector database. + + Args: + client: qdrant_client.QdrantClient | An instance of QdrantClient. + embedding_function: Callable | The embedding function used to generate the vector representation + of the documents. Defaults to FastEmbedEmbeddingFunction. + collection_options: dict | The options for creating the collection. + kwargs: dict | Additional keyword arguments. + """ + self.client: QdrantClient = client if client is not None else QdrantClient(location=":memory:") + self.embedding_function = FastEmbedEmbeddingFunction() or embedding_function + self.collection_options = collection_options + self.content_payload_key = content_payload_key + self.metadata_payload_key = metadata_payload_key + self.type = "qdrant" + + def create_collection(self, collection_name: str, overwrite: bool = False, get_or_create: bool = True) -> None: + """ + Create a collection in the vector database. + + Args: + collection_name: str | The name of the collection. + overwrite: bool | Whether to overwrite the collection if it exists. Default is False. + get_or_create: bool | Whether to get the collection if it exists. Default is True. + + Returns: + Any | The collection object. + """ + embeddings_size = len(self.embedding_function(["test"])[0]) + + if self.client.collection_exists(collection_name) and overwrite: + self.client.delete_collection(collection_name) + + if not self.client.collection_exists(collection_name): + self.client.create_collection( + collection_name, + vectors_config=models.VectorParams(size=embeddings_size, distance=models.Distance.COSINE), + **self.collection_options, + ) + + def get_collection(self, collection_name: str = None): + """ + Get the collection from the vector database. + + Args: + collection_name: str | The name of the collection. + + Returns: + Any | The collection object. + """ + if collection_name is None: + raise ValueError("The collection name is required.") + + return self.client.get_collection(collection_name) + + def delete_collection(self, collection_name: str) -> None: + """Delete the collection from the vector database. + + Args: + collection_name: str | The name of the collection. + + Returns: + Any + """ + return self.client.delete_collection(collection_name) + + def insert_docs(self, docs: List[Document], collection_name: str = None, upsert: bool = False) -> None: + """ + Insert documents into the collection of the vector database. + + Args: + docs: List[Document] | A list of documents. Each document is a TypedDict `Document`. + collection_name: str | The name of the collection. Default is None. + upsert: bool | Whether to update the document if it exists. Default is False. + kwargs: Dict | Additional keyword arguments. + + Returns: + None + """ + if not docs: + return + if any(doc.get("content") is None for doc in docs): + raise ValueError("The document content is required.") + if any(doc.get("id") is None for doc in docs): + raise ValueError("The document id is required.") + + if not upsert and not self._validate_upsert_ids(collection_name, [doc["id"] for doc in docs]): + logger.log("Some IDs already exist. Skipping insert", level=logging.WARN) + + self.client.upsert(collection_name, points=self._documents_to_points(docs)) + + def update_docs(self, docs: List[Document], collection_name: str = None) -> None: + if not docs: + return + if any(doc.get("id") is None for doc in docs): + raise ValueError("The document id is required.") + if any(doc.get("content") is None for doc in docs): + raise ValueError("The document content is required.") + if self._validate_update_ids(collection_name, [doc["id"] for doc in docs]): + return self.client.upsert(collection_name, points=self._documents_to_points(docs)) + + raise ValueError("Some IDs do not exist. Skipping update") + + def delete_docs(self, ids: List[ItemID], collection_name: str = None, **kwargs) -> None: + """ + Delete documents from the collection of the vector database. + + Args: + ids: List[ItemID] | A list of document ids. Each id is a typed `ItemID`. + collection_name: str | The name of the collection. Default is None. + kwargs: Dict | Additional keyword arguments. + + Returns: + None + """ + self.client.delete(collection_name, ids) + + def retrieve_docs( + self, + queries: List[str], + collection_name: str = None, + n_results: int = 10, + distance_threshold: float = 0, + **kwargs, + ) -> QueryResults: + """ + Retrieve documents from the collection of the vector database based on the queries. + + Args: + queries: List[str] | A list of queries. Each query is a string. + collection_name: str | The name of the collection. Default is None. + n_results: int | The number of relevant documents to return. Default is 10. + distance_threshold: float | The threshold for the distance score, only distance smaller than it will be + returned. Don't filter with it if < 0. Default is 0. + kwargs: Dict | Additional keyword arguments. + + Returns: + QueryResults | The query results. Each query result is a list of list of tuples containing the document and + the distance. + """ + embeddings = self.embedding_function(queries) + requests = [ + models.SearchRequest( + vector=embedding, + limit=n_results, + score_threshold=distance_threshold, + with_payload=True, + with_vector=False, + ) + for embedding in embeddings + ] + + batch_results = self.client.search_batch(collection_name, requests) + return [self._scored_points_to_documents(results) for results in batch_results] + + def get_docs_by_ids( + self, ids: List[ItemID] = None, collection_name: str = None, include=True, **kwargs + ) -> List[Document]: + """ + Retrieve documents from the collection of the vector database based on the ids. + + Args: + ids: List[ItemID] | A list of document ids. If None, will return all the documents. Default is None. + collection_name: str | The name of the collection. Default is None. + include: List[str] | The fields to include. Default is True. + If None, will include ["metadatas", "documents"], ids will always be included. + kwargs: dict | Additional keyword arguments. + + Returns: + List[Document] | The results. + """ + if ids is None: + results = self.client.scroll(collection_name=collection_name, with_payload=include, with_vectors=True)[0] + else: + results = self.client.retrieve(collection_name, ids=ids, with_payload=include, with_vectors=True) + return [self._point_to_document(result) for result in results] + + def _point_to_document(self, point) -> Document: + return { + "id": point.id, + "content": point.payload.get(self.content_payload_key, ""), + "metadata": point.payload.get(self.metadata_payload_key, {}), + "embedding": point.vector, + } + + def _points_to_documents(self, points) -> List[Document]: + return [self._point_to_document(point) for point in points] + + def _scored_point_to_document(self, scored_point: models.ScoredPoint) -> Tuple[Document, float]: + return self._point_to_document(scored_point), scored_point.score + + def _documents_to_points(self, documents: List[Document]): + contents = [document["content"] for document in documents] + embeddings = self.embedding_function(contents) + points = [ + models.PointStruct( + id=documents[i]["id"], + vector=embeddings[i], + payload={ + self.content_payload_key: documents[i].get("content"), + self.metadata_payload_key: documents[i].get("metadata"), + }, + ) + for i in range(len(documents)) + ] + return points + + def _scored_points_to_documents(self, scored_points: List[models.ScoredPoint]) -> List[Tuple[Document, float]]: + return [self._scored_point_to_document(scored_point) for scored_point in scored_points] + + def _validate_update_ids(self, collection_name: str, ids: List[str]) -> bool: + """ + Validates all the IDs exist in the collection + """ + retrieved_ids = [ + point.id for point in self.client.retrieve(collection_name, ids=ids, with_payload=False, with_vectors=False) + ] + + if missing_ids := set(ids) - set(retrieved_ids): + logger.log(f"Missing IDs: {missing_ids}. Skipping update", level=logging.WARN) + return False + + return True + + def _validate_upsert_ids(self, collection_name: str, ids: List[str]) -> bool: + """ + Validate none of the IDs exist in the collection + """ + retrieved_ids = [ + point.id for point in self.client.retrieve(collection_name, ids=ids, with_payload=False, with_vectors=False) + ] + + if existing_ids := set(ids) & set(retrieved_ids): + logger.log(f"Existing IDs: {existing_ids}.", level=logging.WARN) + return False + + return True diff --git a/test/agentchat/contrib/vectordb/test_qdrant.py b/test/agentchat/contrib/vectordb/test_qdrant.py new file mode 100644 index 000000000000..c2f4270f2953 --- /dev/null +++ b/test/agentchat/contrib/vectordb/test_qdrant.py @@ -0,0 +1,68 @@ +import os +import sys + +import pytest + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +try: + import uuid + + from qdrant_client import QdrantClient + + from autogen.agentchat.contrib.vectordb.qdrant import QdrantVectorDB +except ImportError: + skip = True +else: + skip = False + + +@pytest.mark.skipif(skip, reason="dependency is not installed") +def test_qdrant(): + # test create collection + client = QdrantClient(location=":memory:") + db = QdrantVectorDB(client=client) + collection_name = uuid.uuid4().hex + db.create_collection(collection_name, overwrite=True, get_or_create=True) + assert client.collection_exists(collection_name) + + # test_delete_collection + db.delete_collection(collection_name) + assert not client.collection_exists(collection_name) + + # test_get_collection + db.create_collection(collection_name, overwrite=True, get_or_create=True) + collection_info = db.get_collection(collection_name) + # Assert default FastEmbed model dimensions + assert collection_info.config.params.vectors.size == 384 + + # test_insert_docs + docs = [{"content": "doc1", "id": 1}, {"content": "doc2", "id": 2}] + db.insert_docs(docs, collection_name, upsert=False) + res = db.get_docs_by_ids([1, 2], collection_name) + assert res[0]["id"] == 1 + assert res[0]["content"] == "doc1" + assert res[1]["id"] == 2 + assert res[1]["content"] == "doc2" + + # test_update_docs and get_docs_by_ids + docs = [{"content": "doc11", "id": 1}, {"content": "doc22", "id": 2}] + db.update_docs(docs, collection_name) + res = db.get_docs_by_ids([1, 2], collection_name) + assert res[0]["id"] == 1 + assert res[0]["content"] == "doc11" + assert res[1]["id"] == 2 + assert res[1]["content"] == "doc22" + + # test_retrieve_docs + queries = ["doc22", "doc11"] + res = db.retrieve_docs(queries, collection_name) + assert [[r[0]["id"] for r in rr] for rr in res] == [[2, 1], [1, 2]] + + # test_delete_docs + db.delete_docs([1], collection_name) + assert db.client.count(collection_name).count == 1 + + +if __name__ == "__main__": + test_qdrant() From 744fcbe299e781988c4370e68ee9fc7171462ed6 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Mon, 15 Jul 2024 23:37:15 +0800 Subject: [PATCH 38/57] Fix websurfer test error (#3138) * Fix assertion error * Update triggers --- .github/workflows/contrib-tests.yml | 2 ++ test/test_browser_utils.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index c8d4d2672fa4..f13bfdbb985c 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -9,6 +9,8 @@ on: paths: - "autogen/**" - "test/agentchat/contrib/**" + - "test/test_browser_utils.py" + - "test/test_retrieve_utils.py" - ".github/workflows/contrib-tests.yml" - "setup.py" diff --git a/test/test_browser_utils.py b/test/test_browser_utils.py index 5551130a6054..659dcce84ae7 100755 --- a/test/test_browser_utils.py +++ b/test/test_browser_utils.py @@ -78,7 +78,7 @@ def test_simple_text_browser(): top_viewport = browser.visit_page(BLOG_POST_URL) assert browser.viewport == top_viewport assert browser.page_title.strip() == BLOG_POST_TITLE.strip() - assert BLOG_POST_STRING in browser.page_content + assert BLOG_POST_STRING in browser.page_content.replace("\n\n", " ").replace("\\", "") # Check if page splitting works approx_pages = math.ceil(len(browser.page_content) / viewport_size) # May be fewer, since it aligns to word breaks From 96146aa9b6533ee67ab7048d328548941df491d0 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 15 Jul 2024 09:51:05 -0700 Subject: [PATCH 39/57] [.Net] update sk version from 1.10.0 to 1.15.1 (#3131) * update sk version * fix sk test error --- dotnet/eng/Version.props | 4 ++-- ...sionTests.ItCreateFunctionContractsFromMethod.approved.txt | 3 +-- ...Tests.ItCreateFunctionContractsFromTestPlugin.approved.txt | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/dotnet/eng/Version.props b/dotnet/eng/Version.props index 572bb22e4676..20be183219e5 100644 --- a/dotnet/eng/Version.props +++ b/dotnet/eng/Version.props @@ -2,8 +2,8 @@ 1.0.0-beta.17 - 1.10.0 - 1.10.0-alpha + 1.15.1 + 1.15.1-alpha 5.0.0 4.3.0 6.0.0 diff --git a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt index 677831d412b7..eb346da3b313 100644 --- a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt +++ b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt @@ -14,8 +14,7 @@ "Name": "message", "Description": "", "ParameterType": "System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", - "IsRequired": true, - "DefaultValue": "" + "IsRequired": true } ], "ReturnType": "System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", diff --git a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt index ee835b1ba081..9ed3c675e4a0 100644 --- a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt +++ b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt @@ -16,8 +16,7 @@ "Name": "newState", "Description": "new state", "ParameterType": "System.Boolean, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", - "IsRequired": true, - "DefaultValue": "" + "IsRequired": true } ], "ReturnType": "System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", From 178bb8d325f1c340d018a47a024193909374db29 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 15 Jul 2024 11:02:30 -0700 Subject: [PATCH 40/57] add cancellation token to transition check lambda (#3132) --- dotnet/src/AutoGen.Core/GroupChat/Graph.cs | 45 +++++++++++++++------- dotnet/test/AutoGen.Tests/WorkflowTest.cs | 4 +- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/dotnet/src/AutoGen.Core/GroupChat/Graph.cs b/dotnet/src/AutoGen.Core/GroupChat/Graph.cs index d6b71e2a3f13..acff955a292c 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/Graph.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/Graph.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace AutoGen.Core; @@ -12,11 +13,7 @@ public class Graph { private readonly List transitions = new List(); - public Graph() - { - } - - public Graph(IEnumerable? transitions) + public Graph(IEnumerable? transitions = null) { if (transitions != null) { @@ -40,13 +37,13 @@ public void AddTransition(Transition transition) /// the from agent /// messages /// A list of agents that the messages can be transit to - public async Task> TransitToNextAvailableAgentsAsync(IAgent fromAgent, IEnumerable messages) + public async Task> TransitToNextAvailableAgentsAsync(IAgent fromAgent, IEnumerable messages, CancellationToken ct = default) { var nextAgents = new List(); var availableTransitions = transitions.FindAll(t => t.From == fromAgent) ?? Enumerable.Empty(); foreach (var transition in availableTransitions) { - if (await transition.CanTransitionAsync(messages)) + if (await transition.CanTransitionAsync(messages, ct)) { nextAgents.Add(transition.To); } @@ -63,7 +60,7 @@ public class Transition { private readonly IAgent _from; private readonly IAgent _to; - private readonly Func, Task>? _canTransition; + private readonly Func, CancellationToken, Task>? _canTransition; ///

/// Create a new instance of . @@ -73,22 +70,44 @@ public class Transition /// from agent /// to agent /// detect if the transition is allowed, default to be always true - internal Transition(IAgent from, IAgent to, Func, Task>? canTransitionAsync = null) + internal Transition(IAgent from, IAgent to, Func, CancellationToken, Task>? canTransitionAsync = null) { _from = from; _to = to; _canTransition = canTransitionAsync; } + /// + /// Create a new instance of without transition condition check. + /// + /// " + public static Transition Create(TFromAgent from, TToAgent to) + where TFromAgent : IAgent + where TToAgent : IAgent + { + return new Transition(from, to, (fromAgent, toAgent, messages, _) => Task.FromResult(true)); + } + /// /// Create a new instance of . /// /// " - public static Transition Create(TFromAgent from, TToAgent to, Func, Task>? canTransitionAsync = null) + public static Transition Create(TFromAgent from, TToAgent to, Func, Task> canTransitionAsync) + where TFromAgent : IAgent + where TToAgent : IAgent + { + return new Transition(from, to, (fromAgent, toAgent, messages, _) => canTransitionAsync.Invoke((TFromAgent)fromAgent, (TToAgent)toAgent, messages)); + } + + /// + /// Create a new instance of with cancellation token. + /// + /// " + public static Transition Create(TFromAgent from, TToAgent to, Func, CancellationToken, Task> canTransitionAsync) where TFromAgent : IAgent where TToAgent : IAgent { - return new Transition(from, to, (fromAgent, toAgent, messages) => canTransitionAsync?.Invoke((TFromAgent)fromAgent, (TToAgent)toAgent, messages) ?? Task.FromResult(true)); + return new Transition(from, to, (fromAgent, toAgent, messages, ct) => canTransitionAsync.Invoke((TFromAgent)fromAgent, (TToAgent)toAgent, messages, ct)); } public IAgent From => _from; @@ -99,13 +118,13 @@ public static Transition Create(TFromAgent from, TToAgent /// Check if the transition is allowed. /// /// messages - public Task CanTransitionAsync(IEnumerable messages) + public Task CanTransitionAsync(IEnumerable messages, CancellationToken ct = default) { if (_canTransition == null) { return Task.FromResult(true); } - return _canTransition(this.From, this.To, messages); + return _canTransition(this.From, this.To, messages, ct); } } diff --git a/dotnet/test/AutoGen.Tests/WorkflowTest.cs b/dotnet/test/AutoGen.Tests/WorkflowTest.cs index d1d12010e39f..1079ec95515a 100644 --- a/dotnet/test/AutoGen.Tests/WorkflowTest.cs +++ b/dotnet/test/AutoGen.Tests/WorkflowTest.cs @@ -17,7 +17,7 @@ public async Task TransitionTestAsync() var alice = new EchoAgent("alice"); var bob = new EchoAgent("bob"); - var aliceToBob = Transition.Create(alice, bob, async (from, to, messages) => + var aliceToBob = Transition.Create(alice, bob, async (from, to, messages, _) => { if (messages.Any(m => m.GetContent() == "Hello")) { @@ -30,7 +30,7 @@ public async Task TransitionTestAsync() var canTransit = await aliceToBob.CanTransitionAsync([]); canTransit.Should().BeFalse(); - canTransit = await aliceToBob.CanTransitionAsync(new[] { new TextMessage(Role.Assistant, "Hello") }); + canTransit = await aliceToBob.CanTransitionAsync([new TextMessage(Role.Assistant, "Hello")]); canTransit.Should().BeTrue(); // if no function is provided, it should always return true From 7205ccc3b9d0c2990d30992c152c5b2b7a074dbf Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 15 Jul 2024 12:20:22 -0700 Subject: [PATCH 41/57] fix build and tests (#3134) --- .../CodeSnippet/CreateAnAgent.cs | 2 +- ...7_Dynamic_GroupChat_Calculate_Fibonacci.cs | 3 +- .../Example09_LMStudio_FunctionCall.cs | 9 +- .../Function/FunctionAttribute.cs | 2 +- .../Template/FunctionCallTemplate.cs | 476 +++++++++++------- .../Template/FunctionCallTemplate.tt | 5 - .../AutoGen.OpenAI.Tests/MathClassTest.cs | 2 +- ...ests.TestFunctionCallTemplate.approved.txt | 7 +- .../FunctionExample.test.cs | 9 +- dotnet/test/AutoGen.Tests/SingleAgentTest.cs | 7 +- dotnet/test/AutoGen.Tests/TwoAgentTest.cs | 2 +- 11 files changed, 320 insertions(+), 204 deletions(-) diff --git a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs index 4833c6195c9d..a103f4ec2d4d 100644 --- a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs +++ b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs @@ -129,7 +129,7 @@ public async Task CodeSnippet5() }, functionMap: new Dictionary>> { - { this.UpperCaseFunction.Name, this.UpperCaseWrapper }, // The wrapper function for the UpperCase function + { this.UpperCaseFunctionContract.Name, this.UpperCaseWrapper }, // The wrapper function for the UpperCase function }); var response = await assistantAgent.SendAsync("hello"); diff --git a/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs b/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs index 6584baa5fae5..72719c606bd2 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs @@ -8,6 +8,7 @@ using AutoGen.Core; using AutoGen.DotnetInteractive; using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; using FluentAssertions; public partial class Example07_Dynamic_GroupChat_Calculate_Fibonacci @@ -138,7 +139,7 @@ public static async Task CreateReviewerAgentAsync() name: "code_reviewer", systemMessage: @"You review code block from coder", config: gpt3Config, - functions: [functions.ReviewCodeBlockFunction], + functions: [functions.ReviewCodeBlockFunctionContract.ToOpenAIFunctionDefinition()], functionMap: new Dictionary>>() { { nameof(ReviewCodeBlock), functions.ReviewCodeBlockWrapper }, diff --git a/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs b/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs index 9a62144df2bd..c9dda27d2e23 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs @@ -5,6 +5,7 @@ using System.Text.Json.Serialization; using AutoGen.Core; using AutoGen.LMStudio; +using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; namespace AutoGen.BasicSample; @@ -69,8 +70,8 @@ public static async Task RunAsync() // And ask agent to response in function call object format using few-shot example object[] functionList = [ - SerializeFunctionDefinition(instance.GetWeatherFunction), - SerializeFunctionDefinition(instance.GoogleSearchFunction) + SerializeFunctionDefinition(instance.GetWeatherFunctionContract.ToOpenAIFunctionDefinition()), + SerializeFunctionDefinition(instance.GetWeatherFunctionContract.ToOpenAIFunctionDefinition()) ]; var functionListString = JsonSerializer.Serialize(functionList, new JsonSerializerOptions { WriteIndented = true }); var lmAgent = new LMStudioAgent( @@ -98,12 +99,12 @@ You are a helpful AI assistant { var arguments = JsonSerializer.Serialize(functionCall.Arguments); // invoke function wrapper - if (functionCall.Name == instance.GetWeatherFunction.Name) + if (functionCall.Name == instance.GetWeatherFunctionContract.Name) { var result = await instance.GetWeatherWrapper(arguments); return new TextMessage(Role.Assistant, result); } - else if (functionCall.Name == instance.GoogleSearchFunction.Name) + else if (functionCall.Name == instance.GetWeatherFunctionContract.Name) { var result = await instance.GoogleSearchWrapper(arguments); return new TextMessage(Role.Assistant, result); diff --git a/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs b/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs index 2c828c26d890..556c16436c63 100644 --- a/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs +++ b/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs @@ -35,7 +35,7 @@ public class FunctionContract /// /// The name of the function. /// - public string? Name { get; set; } + public string Name { get; set; } = null!; /// /// The description of the function. diff --git a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs index cb3574a8b1ea..8eeb117141d8 100644 --- a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs +++ b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs @@ -1,30 +1,31 @@ -๏ปฟ//------------------------------------------------------------------------------ +๏ปฟ// ------------------------------------------------------------------------------ // // This code was generated by a tool. -// +// Runtime Version: 17.0.0.0 +// // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. // -//------------------------------------------------------------------------------ - -namespace AutoGen.SourceGenerator.Template { +// ------------------------------------------------------------------------------ +namespace AutoGen.SourceGenerator.Template +{ using System.Linq; using System.Collections.Generic; using Microsoft.CodeAnalysis; using System; - - internal partial class FunctionCallTemplate : FunctionCallTemplateBase { - - -public string NameSpace {get; set;} -public string ClassName {get; set;} -public IEnumerable FunctionContracts {get; set;} -public bool IsStatic {get; set;} = false; - - - public virtual string TransformText() { - this.GenerationEnvironment = null; + /// + /// Class to produce the template output + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal partial class FunctionCallTemplate : FunctionCallTemplateBase + { + /// + /// Create the template output + /// + public virtual string TransformText() + { + this.Write("๏ปฟ"); this.Write(@"//---------------------- // // This code was generated by a tool. @@ -41,281 +42,402 @@ public virtual string TransformText() { if (!String.IsNullOrEmpty(NameSpace)) { this.Write("namespace "); this.Write(this.ToStringHelper.ToStringWithCulture(NameSpace)); - this.Write("\n{\n"); + this.Write("\r\n{\r\n"); } this.Write(" public partial class "); this.Write(this.ToStringHelper.ToStringWithCulture(ClassName)); - this.Write("\n {\n"); + this.Write("\r\n {\r\n"); foreach (var functionContract in FunctionContracts) { - this.Write("\n private class "); + this.Write("\r\n private class "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionSchemaClassName())); - this.Write("\n {\n"); + this.Write("\r\n {\r\n"); foreach (var parameter in functionContract.Parameters) { if (parameter.IsOptional) { this.Write(" [JsonPropertyName(@\""); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write("\")]\n\t\t\tpublic "); + this.Write("\")]\r\n\t\t\tpublic "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Type)); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); this.Write(" {get; set;} = "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.DefaultValue)); - this.Write(";\n"); + this.Write(";\r\n"); } else { this.Write(" [JsonPropertyName(@\""); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write("\")]\n\t\t\tpublic "); + this.Write("\")]\r\n\t\t\tpublic "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Type)); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write(" {get; set;}\n"); + this.Write(" {get; set;}\r\n"); } } - this.Write(" }\n\n public "); + this.Write(" }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ReturnType)); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionWrapperName())); - this.Write("(string arguments)\n {\n var schema = JsonSerializer.Deserialize<" + - ""); + this.Write("(string arguments)\r\n {\r\n var schema = JsonSerializer.Deserializ" + + "e<"); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionSchemaClassName())); - this.Write(">(\n arguments, \n new JsonSerializerOptions\n " + - " {\n PropertyNamingPolicy = JsonNamingPolicy.CamelCase," + - "\n });\n"); + this.Write(">(\r\n arguments, \r\n new JsonSerializerOptions\r\n " + + " {\r\n PropertyNamingPolicy = JsonNamingPolicy.CamelC" + + "ase,\r\n });\r\n"); var argumentLists = string.Join(", ", functionContract.Parameters.Select(p => $"schema.{p.Name}")); - this.Write("\n return "); + this.Write("\r\n return "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Name)); this.Write("("); this.Write(this.ToStringHelper.ToStringWithCulture(argumentLists)); - this.Write(");\n }\n\n public FunctionContract "); + this.Write(");\r\n }\r\n\r\n public FunctionContract "); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionContractName())); - this.Write("\n {\n get => new FunctionContract\n {\n"); + this.Write("\r\n {\r\n get => new FunctionContract\r\n {\r\n"); if (functionContract.Namespace != null) { this.Write(" Namespace = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Namespace)); - this.Write("\",\n"); + this.Write("\",\r\n"); } if (functionContract.ClassName != null) { this.Write(" ClassName = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ClassName)); - this.Write("\",\n"); + this.Write("\",\r\n"); } if (functionContract.Name != null) { this.Write(" Name = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Name)); - this.Write("\",\n"); + this.Write("\",\r\n"); } if (functionContract.Description != null) { this.Write(" Description = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Description.Replace("\"", "\"\""))); - this.Write("\",\n"); + this.Write("\",\r\n"); } if (functionContract.ReturnType != null) { this.Write(" ReturnType = typeof("); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ReturnType)); - this.Write("),\n"); + this.Write("),\r\n"); } if (functionContract.ReturnDescription != null) { this.Write(" ReturnDescription = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.ReturnDescription)); - this.Write("\",\n"); + this.Write("\",\r\n"); } if (functionContract.Parameters != null) { this.Write(" Parameters = new global::AutoGen.Core.FunctionParameterContract[]" + - "\n {\n"); + "\r\n {\r\n"); foreach (var parameter in functionContract.Parameters) { - this.Write(" new FunctionParameterContract\n {\n"); + this.Write(" new FunctionParameterContract\r\n {\r\n"); if (parameter.Name != null) { this.Write(" Name = @\""); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Name)); - this.Write("\",\n"); + this.Write("\",\r\n"); } if (parameter.Description != null) { this.Write(" Description = @\""); - this.Write(this.ToStringHelper.ToStringWithCulture( parameter.Description.Replace("\"", "\"\"") )); - this.Write("\",\n"); + this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Description.Replace("\"", "\"\""))); + this.Write("\",\r\n"); } if (parameter.Type != null) { this.Write(" ParameterType = typeof("); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Type)); - this.Write("),\n"); + this.Write("),\r\n"); } this.Write(" IsRequired = "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.IsOptional ? "false" : "true")); - this.Write(",\n"); + this.Write(",\r\n"); if (parameter.DefaultValue != null) { this.Write(" DefaultValue = "); this.Write(this.ToStringHelper.ToStringWithCulture(parameter.DefaultValue)); - this.Write(",\n"); + this.Write(",\r\n"); } - this.Write(" },\n"); + this.Write(" },\r\n"); } - this.Write(" },\n"); + this.Write(" },\r\n"); } - this.Write(" };\n }\n\n public global::Azure.AI.OpenAI.FunctionDefiniti" + - "on "); - this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionDefinitionName())); - this.Write("\n {\n get => this."); - this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionContractName())); - this.Write(".ToOpenAIFunctionDefinition();\n }\n"); + this.Write(" };\r\n }\r\n"); } - this.Write(" }\n"); + this.Write(" }\r\n"); if (!String.IsNullOrEmpty(NameSpace)) { - this.Write("}\n"); + this.Write("}\r\n"); } - this.Write("\n"); + this.Write("\r\n"); return this.GenerationEnvironment.ToString(); } - - public virtual void Initialize() { - } + +public string NameSpace {get; set;} +public string ClassName {get; set;} +public IEnumerable FunctionContracts {get; set;} +public bool IsStatic {get; set;} = false; + } - - public class FunctionCallTemplateBase { - - private global::System.Text.StringBuilder builder; - - private global::System.Collections.Generic.IDictionary session; - - private global::System.CodeDom.Compiler.CompilerErrorCollection errors; - - private string currentIndent = string.Empty; - - private global::System.Collections.Generic.Stack indents; - - private ToStringInstanceHelper _toStringHelper = new ToStringInstanceHelper(); - - public virtual global::System.Collections.Generic.IDictionary Session { - get { - return this.session; + #region Base class + /// + /// Base class for this transformation + /// + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")] + internal class FunctionCallTemplateBase + { + #region Fields + private global::System.Text.StringBuilder generationEnvironmentField; + private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField; + private global::System.Collections.Generic.List indentLengthsField; + private string currentIndentField = ""; + private bool endsWithNewline; + private global::System.Collections.Generic.IDictionary sessionField; + #endregion + #region Properties + /// + /// The string builder that generation-time code is using to assemble generated output + /// + public System.Text.StringBuilder GenerationEnvironment + { + get + { + if ((this.generationEnvironmentField == null)) + { + this.generationEnvironmentField = new global::System.Text.StringBuilder(); + } + return this.generationEnvironmentField; } - set { - this.session = value; + set + { + this.generationEnvironmentField = value; } } - - public global::System.Text.StringBuilder GenerationEnvironment { - get { - if ((this.builder == null)) { - this.builder = new global::System.Text.StringBuilder(); + /// + /// The error collection for the generation process + /// + public System.CodeDom.Compiler.CompilerErrorCollection Errors + { + get + { + if ((this.errorsField == null)) + { + this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection(); } - return this.builder; - } - set { - this.builder = value; + return this.errorsField; } } - - protected global::System.CodeDom.Compiler.CompilerErrorCollection Errors { - get { - if ((this.errors == null)) { - this.errors = new global::System.CodeDom.Compiler.CompilerErrorCollection(); + /// + /// A list of the lengths of each indent that was added with PushIndent + /// + private System.Collections.Generic.List indentLengths + { + get + { + if ((this.indentLengthsField == null)) + { + this.indentLengthsField = new global::System.Collections.Generic.List(); } - return this.errors; + return this.indentLengthsField; } } - - public string CurrentIndent { - get { - return this.currentIndent; + /// + /// Gets the current indent we use when adding lines to the output + /// + public string CurrentIndent + { + get + { + return this.currentIndentField; } } - - private global::System.Collections.Generic.Stack Indents { - get { - if ((this.indents == null)) { - this.indents = new global::System.Collections.Generic.Stack(); - } - return this.indents; + /// + /// Current transformation session + /// + public virtual global::System.Collections.Generic.IDictionary Session + { + get + { + return this.sessionField; } - } - - public ToStringInstanceHelper ToStringHelper { - get { - return this._toStringHelper; + set + { + this.sessionField = value; } } - - public void Error(string message) { - this.Errors.Add(new global::System.CodeDom.Compiler.CompilerError(null, -1, -1, null, message)); + #endregion + #region Transform-time helpers + /// + /// Write text directly into the generated output + /// + public void Write(string textToAppend) + { + if (string.IsNullOrEmpty(textToAppend)) + { + return; + } + // If we're starting off, or if the previous text ended with a newline, + // we have to append the current indent first. + if (((this.GenerationEnvironment.Length == 0) + || this.endsWithNewline)) + { + this.GenerationEnvironment.Append(this.currentIndentField); + this.endsWithNewline = false; + } + // Check if the current text ends with a newline + if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture)) + { + this.endsWithNewline = true; + } + // This is an optimization. If the current indent is "", then we don't have to do any + // of the more complex stuff further down. + if ((this.currentIndentField.Length == 0)) + { + this.GenerationEnvironment.Append(textToAppend); + return; + } + // Everywhere there is a newline in the text, add an indent after it + textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField)); + // If the text ends with a newline, then we should strip off the indent added at the very end + // because the appropriate indent will be added when the next time Write() is called + if (this.endsWithNewline) + { + this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length)); + } + else + { + this.GenerationEnvironment.Append(textToAppend); + } } - - public void Warning(string message) { - global::System.CodeDom.Compiler.CompilerError val = new global::System.CodeDom.Compiler.CompilerError(null, -1, -1, null, message); - val.IsWarning = true; - this.Errors.Add(val); + /// + /// Write text directly into the generated output + /// + public void WriteLine(string textToAppend) + { + this.Write(textToAppend); + this.GenerationEnvironment.AppendLine(); + this.endsWithNewline = true; } - - public string PopIndent() { - if ((this.Indents.Count == 0)) { - return string.Empty; - } - int lastPos = (this.currentIndent.Length - this.Indents.Pop()); - string last = this.currentIndent.Substring(lastPos); - this.currentIndent = this.currentIndent.Substring(0, lastPos); - return last; + /// + /// Write formatted text directly into the generated output + /// + public void Write(string format, params object[] args) + { + this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); } - - public void PushIndent(string indent) { - this.Indents.Push(indent.Length); - this.currentIndent = (this.currentIndent + indent); + /// + /// Write formatted text directly into the generated output + /// + public void WriteLine(string format, params object[] args) + { + this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args)); } - - public void ClearIndent() { - this.currentIndent = string.Empty; - this.Indents.Clear(); + /// + /// Raise an error + /// + public void Error(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + this.Errors.Add(error); } - - public void Write(string textToAppend) { - this.GenerationEnvironment.Append(textToAppend); + /// + /// Raise a warning + /// + public void Warning(string message) + { + System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError(); + error.ErrorText = message; + error.IsWarning = true; + this.Errors.Add(error); } - - public void Write(string format, params object[] args) { - this.GenerationEnvironment.AppendFormat(format, args); + /// + /// Increase the indent + /// + public void PushIndent(string indent) + { + if ((indent == null)) + { + throw new global::System.ArgumentNullException("indent"); + } + this.currentIndentField = (this.currentIndentField + indent); + this.indentLengths.Add(indent.Length); } - - public void WriteLine(string textToAppend) { - this.GenerationEnvironment.Append(this.currentIndent); - this.GenerationEnvironment.AppendLine(textToAppend); + /// + /// Remove the last indent that was added with PushIndent + /// + public string PopIndent() + { + string returnValue = ""; + if ((this.indentLengths.Count > 0)) + { + int indentLength = this.indentLengths[(this.indentLengths.Count - 1)]; + this.indentLengths.RemoveAt((this.indentLengths.Count - 1)); + if ((indentLength > 0)) + { + returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength)); + this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength)); + } + } + return returnValue; } - - public void WriteLine(string format, params object[] args) { - this.GenerationEnvironment.Append(this.currentIndent); - this.GenerationEnvironment.AppendFormat(format, args); - this.GenerationEnvironment.AppendLine(); + /// + /// Remove any indentation + /// + public void ClearIndent() + { + this.indentLengths.Clear(); + this.currentIndentField = ""; } - - public class ToStringInstanceHelper { - - private global::System.IFormatProvider formatProvider = global::System.Globalization.CultureInfo.InvariantCulture; - - public global::System.IFormatProvider FormatProvider { - get { - return this.formatProvider; + #endregion + #region ToString Helpers + /// + /// Utility class to produce culture-oriented representation of an object as a string. + /// + public class ToStringInstanceHelper + { + private System.IFormatProvider formatProviderField = global::System.Globalization.CultureInfo.InvariantCulture; + /// + /// Gets or sets format provider to be used by ToStringWithCulture method. + /// + public System.IFormatProvider FormatProvider + { + get + { + return this.formatProviderField ; } - set { - if ((value != null)) { - this.formatProvider = value; + set + { + if ((value != null)) + { + this.formatProviderField = value; } } } - - public string ToStringWithCulture(object objectToConvert) { - if ((objectToConvert == null)) { + /// + /// This is called from the compile/run appdomain to convert objects within an expression block to a string + /// + public string ToStringWithCulture(object objectToConvert) + { + if ((objectToConvert == null)) + { throw new global::System.ArgumentNullException("objectToConvert"); } - global::System.Type type = objectToConvert.GetType(); - global::System.Type iConvertibleType = typeof(global::System.IConvertible); - if (iConvertibleType.IsAssignableFrom(type)) { - return ((global::System.IConvertible)(objectToConvert)).ToString(this.formatProvider); + System.Type t = objectToConvert.GetType(); + System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] { + typeof(System.IFormatProvider)}); + if ((method == null)) + { + return objectToConvert.ToString(); } - global::System.Reflection.MethodInfo methInfo = type.GetMethod("ToString", new global::System.Type[] { - iConvertibleType}); - if ((methInfo != null)) { - return ((string)(methInfo.Invoke(objectToConvert, new object[] { - this.formatProvider}))); + else + { + return ((string)(method.Invoke(objectToConvert, new object[] { + this.formatProviderField }))); } - return objectToConvert.ToString(); } } + private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper(); + /// + /// Helper to produce culture-oriented representation of an object as a string + /// + public ToStringInstanceHelper ToStringHelper + { + get + { + return this.toStringHelperField; + } + } + #endregion } + #endregion } diff --git a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt index c7a13ce1fe92..dc41f0af9d70 100644 --- a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt +++ b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt @@ -96,11 +96,6 @@ namespace <#=NameSpace#> <#}#> }; } - - public global::Azure.AI.OpenAI.FunctionDefinition <#=functionContract.GetFunctionDefinitionName()#> - { - get => this.<#=functionContract.GetFunctionContractName()#>.ToOpenAIFunctionDefinition(); - } <#}#> } <#if (!String.IsNullOrEmpty(NameSpace)) {#> diff --git a/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs b/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs index aae314ff773e..01af3d4646c4 100644 --- a/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs +++ b/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs @@ -110,7 +110,7 @@ public async Task OpenAIAgentMathChatTestAsync() functions: [this.UpdateProgressFunctionContract], functionMap: new Dictionary>> { - { this.UpdateProgressFunction.Name!, this.UpdateProgressWrapper }, + { this.UpdateProgressFunctionContract.Name, this.UpdateProgressWrapper }, }); var admin = new OpenAIChatAgent( openAIClient: openaiClient, diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt b/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt index a5ffa3c8248d..f223d3124ddd 100644 --- a/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt @@ -1,4 +1,4 @@ -๏ปฟ//---------------------- +๏ปฟ๏ปฟ//---------------------- // // This code was generated by a tool. // @@ -61,11 +61,6 @@ namespace AutoGen.SourceGenerator.Tests }, }; } - - public global::Azure.AI.OpenAI.FunctionDefinition AddAsyncFunction - { - get => this.AddAsyncFunctionContract.ToOpenAIFunctionDefinition(); - } } } diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs index f7b90e0b96ff..0096f2c157ce 100644 --- a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs @@ -5,6 +5,7 @@ using ApprovalTests; using ApprovalTests.Namers; using ApprovalTests.Reporters; +using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; using FluentAssertions; using Xunit; @@ -29,7 +30,7 @@ public void Add_Test() }; this.VerifyFunction(functionExamples.AddWrapper, args, 3); - this.VerifyFunctionDefinition(functionExamples.AddFunction); + this.VerifyFunctionDefinition(functionExamples.AddFunctionContract.ToOpenAIFunctionDefinition()); } [Fact] @@ -41,7 +42,7 @@ public void Sum_Test() }; this.VerifyFunction(functionExamples.SumWrapper, args, 6.0); - this.VerifyFunctionDefinition(functionExamples.SumFunction); + this.VerifyFunctionDefinition(functionExamples.SumFunctionContract.ToOpenAIFunctionDefinition()); } [Fact] @@ -57,7 +58,7 @@ public async Task DictionaryToString_Test() }; await this.VerifyAsyncFunction(functionExamples.DictionaryToStringAsyncWrapper, args, JsonSerializer.Serialize(args.xargs, jsonSerializerOptions)); - this.VerifyFunctionDefinition(functionExamples.DictionaryToStringAsyncFunction); + this.VerifyFunctionDefinition(functionExamples.DictionaryToStringAsyncFunctionContract.ToOpenAIFunctionDefinition()); } [Fact] @@ -96,7 +97,7 @@ public void Query_Test() }; this.VerifyFunction(functionExamples.QueryWrapper, args, new[] { "hello", "hello", "hello" }); - this.VerifyFunctionDefinition(functionExamples.QueryFunction); + this.VerifyFunctionDefinition(functionExamples.QueryFunctionContract.ToOpenAIFunctionDefinition()); } [UseReporter(typeof(DiffReporter))] diff --git a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs index 64bdc062eb73..b545bbdbe864 100644 --- a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs +++ b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using AutoGen.LMStudio; using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; using FluentAssertions; using Xunit; @@ -64,7 +65,7 @@ public async Task GPTAgentVisionTestAsync() systemMessage: "You are a helpful AI assistant, return highest label from conversation", config: gpt3Config, temperature: 0, - functions: new[] { this.GetHighestLabelFunction }, + functions: new[] { this.GetHighestLabelFunctionContract.ToOpenAIFunctionDefinition() }, functionMap: new Dictionary>> { { nameof(GetHighestLabel), this.GetHighestLabelWrapper }, @@ -116,7 +117,7 @@ public async Task GPTAgentVisionTestAsync() public async Task GPTFunctionCallAgentTestAsync() { var config = this.CreateAzureOpenAIGPT35TurboConfig(); - var agentWithFunction = new GPTAgent("gpt", "You are a helpful AI assistant", config, 0, functions: new[] { this.EchoAsyncFunction }); + var agentWithFunction = new GPTAgent("gpt", "You are a helpful AI assistant", config, 0, functions: new[] { this.EchoAsyncFunctionContract.ToOpenAIFunctionDefinition() }); await EchoFunctionCallTestAsync(agentWithFunction); } @@ -233,7 +234,7 @@ public async Task GPTAgentFunctionCallSelfExecutionTestAsync() systemMessage: "You are a helpful AI assistant", config: config, temperature: 0, - functions: new[] { this.EchoAsyncFunction }, + functions: new[] { this.EchoAsyncFunctionContract.ToOpenAIFunctionDefinition() }, functionMap: new Dictionary>> { { nameof(EchoAsync), this.EchoAsyncWrapper }, diff --git a/dotnet/test/AutoGen.Tests/TwoAgentTest.cs b/dotnet/test/AutoGen.Tests/TwoAgentTest.cs index 90c1bfa9a148..100a22c04a71 100644 --- a/dotnet/test/AutoGen.Tests/TwoAgentTest.cs +++ b/dotnet/test/AutoGen.Tests/TwoAgentTest.cs @@ -56,7 +56,7 @@ public async Task TwoAgentWeatherChatTestAsync() name: "user", functionMap: new Dictionary>> { - { this.GetWeatherFunction.Name, this.GetWeatherWrapper }, + { this.GetWeatherFunctionContract.Name, this.GetWeatherWrapper }, }) .RegisterMiddleware(async (msgs, option, agent, ct) => { From 970c1961ddf799238cf04e10bd512bb3db0ec225 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 15 Jul 2024 12:33:10 -0700 Subject: [PATCH 42/57] [.Net] update dotnet-ci and dotnet-release to use 8.0.x version when setting up .NET. And enable format check (#3136) * use 8.0.x versin * enable format check * change file header * apply code format * add instructions in ci to fix format error * add comment back --- .github/workflows/dotnet-build.yml | 7 ++++++- .github/workflows/dotnet-release.yml | 2 +- .../AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs | 2 +- .../Create_Anthropic_Agent_With_Tool.cs | 2 +- .../CodeSnippet/MiddlewareAgentCodeSnippet.cs | 6 ++---- .../Example04_Dynamic_GroupChat_Coding_Task.cs | 4 ++++ .../Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs | 4 ++++ .../Example13_OpenAIAgent_JsonMode.cs | 4 +++- .../Example15_GPT4V_BinaryDataImageMessage.cs | 2 ++ ...xample16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs | 3 ++- .../GettingStart/Dynamic_Group_Chat.cs | 2 +- dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs | 3 ++- dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs | 2 +- .../src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs | 5 ++++- dotnet/src/AutoGen.Anthropic/AnthropicClient.cs | 2 ++ .../AutoGen.Anthropic/Converters/ContentBaseConverter.cs | 6 ++---- dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs | 2 +- .../src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs | 5 +++-- dotnet/src/AutoGen.Anthropic/DTO/Content.cs | 2 +- dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs | 2 +- dotnet/src/AutoGen.Anthropic/DTO/Tool.cs | 2 +- dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs | 2 +- .../src/AutoGen.Core/Message/ToolCallAggregateMessage.cs | 2 +- dotnet/src/AutoGen.Gemini/IGeminiClient.cs | 2 +- dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs | 2 +- dotnet/src/AutoGen.Ollama/DTOs/Message.cs | 2 +- .../AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs | 2 +- .../Embeddings/OllamaTextEmbeddingService.cs | 2 +- .../AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs | 2 +- .../AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs | 2 +- .../SourceGeneratorFunctionContract.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs | 2 +- .../src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs | 2 +- .../AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs | 2 +- .../test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs | 7 ++++++- .../AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs | 2 +- dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs | 8 ++++---- .../test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs | 2 +- .../OllamaTextEmbeddingServiceTests.cs | 2 +- .../FunctionCallTemplateEncodingTests.cs | 8 +++++--- dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs | 4 +++- dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs | 2 +- 42 files changed, 80 insertions(+), 50 deletions(-) diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml index f4074b061693..7e50025917de 100644 --- a/.github/workflows/dotnet-build.yml +++ b/.github/workflows/dotnet-build.yml @@ -56,11 +56,16 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - global-json-file: dotnet/global.json + dotnet-version: '8.0.x' - name: Restore dependencies run: | # dotnet nuget add source --name dotnet-tool https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json --configfile NuGet.config dotnet restore -bl + - name: Format check + run: | + echo "Format check" + echo "If you see any error in this step, please run 'dotnet format' locally to format the code." + dotnet format --verify-no-changes -v diag --no-restore - name: Build run: | echo "Build AutoGen" diff --git a/.github/workflows/dotnet-release.yml b/.github/workflows/dotnet-release.yml index 2877d058377b..aacfd115bb7e 100644 --- a/.github/workflows/dotnet-release.yml +++ b/.github/workflows/dotnet-release.yml @@ -32,7 +32,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - global-json-file: dotnet/global.json + dotnet-version: '8.0.x' - name: Restore dependencies run: | dotnet restore -bl diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs index 031e50685488..6f32c3cb4a21 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// AnthropicSamples.cs +// Create_Anthropic_Agent.cs using AutoGen.Anthropic.Extensions; using AutoGen.Anthropic.Utils; diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs index 26bd32dd12d5..0324a39ffa59 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// Single_Anthropic_Tool.cs +// Create_Anthropic_Agent_With_Tool.cs using AutoGen.Anthropic.DTO; using AutoGen.Anthropic.Extensions; diff --git a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs index 795606ebc85c..1b5a9a903207 100644 --- a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs +++ b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs @@ -20,8 +20,7 @@ public async Task CreateMiddlewareAgentAsync() var middlewareAgent = new MiddlewareAgent(innerAgent: agent); middlewareAgent.Use(async (messages, options, agent, ct) => { - var lastMessage = messages.Last() as TextMessage; - if (lastMessage != null && lastMessage.Content.Contains("Hello World")) + if (messages.Last() is TextMessage lastMessage && lastMessage.Content.Contains("Hello World")) { lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; return lastMessage; @@ -39,8 +38,7 @@ public async Task CreateMiddlewareAgentAsync() #region register_middleware_agent middlewareAgent = agent.RegisterMiddleware(async (messages, options, agent, ct) => { - var lastMessage = messages.Last() as TextMessage; - if (lastMessage != null && lastMessage.Content.Contains("Hello World")) + if (messages.Last() is TextMessage lastMessage && lastMessage.Content.Contains("Hello World")) { lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; return lastMessage; diff --git a/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs b/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs index 47dd8ce66c90..216059928408 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs @@ -17,14 +17,18 @@ public static async Task RunAsync() // setup dotnet interactive var workDir = Path.Combine(Path.GetTempPath(), "InteractiveService"); if (!Directory.Exists(workDir)) + { Directory.CreateDirectory(workDir); + } using var service = new InteractiveService(workDir); var dotnetInteractiveFunctions = new DotnetInteractiveFunction(service); var result = Path.Combine(workDir, "result.txt"); if (File.Exists(result)) + { File.Delete(result); + } await service.StartAsync(workDir, default); diff --git a/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs b/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs index 72719c606bd2..004e0f055449 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs @@ -225,7 +225,9 @@ public static async Task RunWorkflowAsync() long the39thFibonacciNumber = 63245986; var workDir = Path.Combine(Path.GetTempPath(), "InteractiveService"); if (!Directory.Exists(workDir)) + { Directory.CreateDirectory(workDir); + } using var service = new InteractiveService(workDir); var dotnetInteractiveFunctions = new DotnetInteractiveFunction(service); @@ -329,7 +331,9 @@ public static async Task RunAsync() long the39thFibonacciNumber = 63245986; var workDir = Path.Combine(Path.GetTempPath(), "InteractiveService"); if (!Directory.Exists(workDir)) + { Directory.CreateDirectory(workDir); + } using var service = new InteractiveService(workDir); var dotnetInteractiveFunctions = new DotnetInteractiveFunction(service); diff --git a/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs b/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs index 9e5b91ecc120..596ab08d02a1 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs @@ -1,3 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Example13_OpenAIAgent_JsonMode.cs -// this example has been moved to https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs \ No newline at end of file + +// this example has been moved to https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs + diff --git a/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs b/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs index 788122d3f383..dee9915511d6 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs @@ -50,7 +50,9 @@ private static void AddMessagesFromResource(string imageResourcePath, List CreateChatCompletionsAsync(ChatComplet var responseStream = await httpResponseMessage.Content.ReadAsStreamAsync(); if (httpResponseMessage.IsSuccessStatusCode) + { return await DeserializeResponseAsync(responseStream, cancellationToken); + } ErrorResponse res = await DeserializeResponseAsync(responseStream, cancellationToken); throw new Exception(res.Error?.Message); diff --git a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs index b41a761dc4d3..3e620f934c28 100644 --- a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs +++ b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs @@ -1,12 +1,10 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// ContentConverter.cs - -using AutoGen.Anthropic.DTO; - +// ContentBaseConverter.cs using System; using System.Text.Json; using System.Text.Json.Serialization; +using AutoGen.Anthropic.DTO; namespace AutoGen.Anthropic.Converters; public sealed class ContentBaseConverter : JsonConverter diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs index b18461e697bc..463ee7fc2595 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs @@ -1,7 +1,7 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // ChatCompletionRequest.cs -using System.Text.Json.Serialization; using System.Collections.Generic; +using System.Text.Json.Serialization; namespace AutoGen.Anthropic.DTO; diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs index 2c6fa100fd63..fc33aa0e26b1 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs @@ -1,10 +1,11 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// ChatCompletionResponse.cs -namespace AutoGen.Anthropic.DTO; using System.Collections.Generic; using System.Text.Json.Serialization; +namespace AutoGen.Anthropic.DTO; public class ChatCompletionResponse { [JsonPropertyName("content")] diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs index ee7a745a1416..353cf6ae824b 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Content.cs using System.Text.Json.Nodes; diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs b/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs index d02a8f6d1cfc..1a94334c88ff 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // ErrorResponse.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs index 41c20dc2a42d..2a46bc42a35b 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Tool.cs using System.Collections.Generic; diff --git a/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs b/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs index e70572cbddf2..6fd70cb4ee3e 100644 --- a/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs +++ b/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// Constants.cs +// AnthropicConstants.cs namespace AutoGen.Anthropic.Utils; diff --git a/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs b/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs index 7781b785ef8c..7d46d56135aa 100644 --- a/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// FunctionCallAggregateMessage.cs +// ToolCallAggregateMessage.cs using System.Collections.Generic; diff --git a/dotnet/src/AutoGen.Gemini/IGeminiClient.cs b/dotnet/src/AutoGen.Gemini/IGeminiClient.cs index 2e209e02b030..d391a4508398 100644 --- a/dotnet/src/AutoGen.Gemini/IGeminiClient.cs +++ b/dotnet/src/AutoGen.Gemini/IGeminiClient.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// IVertexGeminiClient.cs +// IGeminiClient.cs using System.Collections.Generic; using System.Threading; diff --git a/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs b/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs index c54f2280dfd3..12a11993cd69 100644 --- a/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs +++ b/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// IGeminiClient.cs +// VertexGeminiClient.cs using System.Collections.Generic; using System.Threading; diff --git a/dotnet/src/AutoGen.Ollama/DTOs/Message.cs b/dotnet/src/AutoGen.Ollama/DTOs/Message.cs index 2e0d891cc61e..75f622ff7f04 100644 --- a/dotnet/src/AutoGen.Ollama/DTOs/Message.cs +++ b/dotnet/src/AutoGen.Ollama/DTOs/Message.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// ChatResponseUpdate.cs +// Message.cs using System.Collections.Generic; using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs b/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs index 5ce0dc8cc40a..cce6dbb83076 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // ITextEmbeddingService.cs using System.Threading; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs b/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs index 2e431e7bcb81..ea4993eb813f 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // OllamaTextEmbeddingService.cs using System; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs index 7f2531c522ad..d776b183db0b 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // TextEmbeddingsRequest.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs index 580059c033b5..f3ce64b7032f 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // TextEmbeddingsResponse.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs b/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs index 24e42affa3bd..aa4980379f4f 100644 --- a/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs +++ b/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// FunctionContract.cs +// SourceGeneratorFunctionContract.cs namespace AutoGen.SourceGenerator { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs index 3cad9ae36336..24cbb72fb6c0 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// OpenAIChatCompletionOption.cs +// OpenAIMessage.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs index 3c188b865b77..7a49851446be 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// +// OpenAIStreamOptions.cs using System.Text.Json.Serialization; diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs index 552408f1d056..085917d419e9 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs @@ -188,7 +188,7 @@ public async Task AnthropicAgentFunctionCallMessageTest() IMessage[] chatHistory = [ new TextMessage(Role.User, "what's the weather in Philadelphia?"), new ToolCallMessage([toolCall], from: "assistant"), - new ToolCallResultMessage([toolCall], from: "user" ), + new ToolCallResultMessage([toolCall], from: "user"), ]; var reply = await agent.SendAsync(chatHistory: chatHistory); diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs index 66b7d007758a..102e48b9b8ac 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs @@ -1,4 +1,7 @@ -๏ปฟusing System.Text; +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// AnthropicClientTest.cs + +using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; @@ -59,7 +62,9 @@ public async Task AnthropicClientStreamingChatCompletionTestAsync() foreach (ChatCompletionResponse result in results) { if (result.Delta is not null && !string.IsNullOrEmpty(result.Delta.Text)) + { sb.Append(result.Delta.Text); + } } string resultContent = sb.ToString(); diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs index 5f1c0971bf71..8b5466e3a519 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// AnthropicTestFunctions.cs +// AnthropicTestFunctionCalls.cs using System.Text.Json; using System.Text.Json.Serialization; diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs index 6d095845f803..c076aee18376 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs @@ -1,13 +1,13 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // GeminiAgentTests.cs -using AutoGen.Tests; -using Google.Cloud.AIPlatform.V1; using AutoGen.Core; -using FluentAssertions; using AutoGen.Gemini.Extension; -using static Google.Cloud.AIPlatform.V1.Part; +using AutoGen.Tests; +using FluentAssertions; +using Google.Cloud.AIPlatform.V1; using Xunit.Abstractions; +using static Google.Cloud.AIPlatform.V1.Part; namespace AutoGen.Gemini.Tests; public class GeminiAgentTests diff --git a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs index 8063b7077035..fba97aa522d5 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// GeminiVertexClientTests.cs +// VertexGeminiClientTests.cs using AutoGen.Tests; using FluentAssertions; diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs index 06522bdd8238..b7186a3c6ebc 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // OllamaTextEmbeddingServiceTests.cs using AutoGen.Tests; diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs index 8ca6e31a8a40..0b2e211c6386 100644 --- a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs @@ -1,7 +1,9 @@ -// Using directives +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// FunctionCallTemplateEncodingTests.cs + using System.Text.Json; // Needed for JsonSerializer -using Xunit; // Needed for Fact and Assert using AutoGen.SourceGenerator.Template; // Needed for FunctionCallTemplate +using Xunit; // Needed for Fact and Assert namespace AutoGen.SourceGenerator.Tests { @@ -89,4 +91,4 @@ public void ParameterDescription_Should_Encode_DoubleQuotes() Assert.Contains("Description = @\"This is a \"\"parameter\"\" description\"", result); } } -} \ No newline at end of file +} diff --git a/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs b/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs index 77e2c99dcd13..7eeea6743f04 100644 --- a/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs +++ b/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs @@ -1,4 +1,6 @@ -๏ปฟ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// GraphTests.cs + using Xunit; namespace AutoGen.Tests diff --git a/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs index 64d48d2a00be..6f9d217b50c2 100644 --- a/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs +++ b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs @@ -1,5 +1,5 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. -// OpenAIChatCompletionMiddlewareTests.cs +// EchoAgent.cs using System.Runtime.CompilerServices; using AutoGen.Core; From 5292b816ad739be5406b34b0c36165b7aba62272 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 15 Jul 2024 16:13:31 -0700 Subject: [PATCH 43/57] update (#3144) (#3145) --- dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj b/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj index cd24d5930b36..c5b720764761 100644 --- a/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj +++ b/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj @@ -6,6 +6,17 @@ $(NoWarn);CS1591;CS1573 + + + + + + AutoGen.WebAPI + + Turn an `AutoGen.Core.IAgent` into a RESTful API. + + + From 6ef5c3027dee7093b49ef62e2bd11ae1760dd3c1 Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Tue, 16 Jul 2024 14:44:48 +0800 Subject: [PATCH 44/57] Update qdrant notebook for new qdrant vectordb (#3140) * Add qdrant notebook, rename notebooks * Revert changes of pgvector notebook * Fix assertion error --- autogen/agentchat/contrib/vectordb/base.py | 17 +- autogen/agentchat/contrib/vectordb/qdrant.py | 8 +- ... => agentchat_RetrieveChat_pgvector.ipynb} | 0 ...nb => agentchat_RetrieveChat_qdrant.ipynb} | 393 +++++++++--------- .../blog/2023-10-18-RetrieveChat/index.mdx | 2 +- website/docs/Examples.md | 2 +- website/docs/ecosystem/pgvector.md | 2 +- .../installation/Optional-Dependencies.md | 2 +- website/docs/topics/retrieval_augmentation.md | 4 +- 9 files changed, 220 insertions(+), 210 deletions(-) rename notebook/{agentchat_pgvector_RetrieveChat.ipynb => agentchat_RetrieveChat_pgvector.ipynb} (100%) rename notebook/{agentchat_qdrant_RetrieveChat.ipynb => agentchat_RetrieveChat_qdrant.ipynb} (50%) diff --git a/autogen/agentchat/contrib/vectordb/base.py b/autogen/agentchat/contrib/vectordb/base.py index 1531a6494cef..20b6376d01d9 100644 --- a/autogen/agentchat/contrib/vectordb/base.py +++ b/autogen/agentchat/contrib/vectordb/base.py @@ -1,4 +1,16 @@ -from typing import Any, List, Mapping, Optional, Protocol, Sequence, Tuple, TypedDict, Union, runtime_checkable +from typing import ( + Any, + Callable, + List, + Mapping, + Optional, + Protocol, + Sequence, + Tuple, + TypedDict, + Union, + runtime_checkable, +) Metadata = Union[Mapping[str, Any], None] Vector = Union[Sequence[float], Sequence[int]] @@ -49,6 +61,9 @@ class VectorDB(Protocol): active_collection: Any = None type: str = "" + embedding_function: Optional[Callable[[List[str]], List[List[float]]]] = ( + None # embeddings = embedding_function(sentences) + ) def create_collection(self, collection_name: str, overwrite: bool = False, get_or_create: bool = True) -> Any: """ diff --git a/autogen/agentchat/contrib/vectordb/qdrant.py b/autogen/agentchat/contrib/vectordb/qdrant.py index 479d7a485270..398734eb0334 100644 --- a/autogen/agentchat/contrib/vectordb/qdrant.py +++ b/autogen/agentchat/contrib/vectordb/qdrant.py @@ -92,7 +92,7 @@ def __init__( collection_options: dict | The options for creating the collection. kwargs: dict | Additional keyword arguments. """ - self.client: QdrantClient = client if client is not None else QdrantClient(location=":memory:") + self.client: QdrantClient = client or QdrantClient(location=":memory:") self.embedding_function = FastEmbedEmbeddingFunction() or embedding_function self.collection_options = collection_options self.content_payload_key = content_payload_key @@ -102,6 +102,10 @@ def __init__( def create_collection(self, collection_name: str, overwrite: bool = False, get_or_create: bool = True) -> None: """ Create a collection in the vector database. + Case 1. if the collection does not exist, create the collection. + Case 2. the collection exists, if overwrite is True, it will overwrite the collection. + Case 3. the collection exists and overwrite is False, if get_or_create is True, it will get the collection, + otherwise it raise a ValueError. Args: collection_name: str | The name of the collection. @@ -122,6 +126,8 @@ def create_collection(self, collection_name: str, overwrite: bool = False, get_o vectors_config=models.VectorParams(size=embeddings_size, distance=models.Distance.COSINE), **self.collection_options, ) + elif not get_or_create: + raise ValueError(f"Collection {collection_name} already exists.") def get_collection(self, collection_name: str = None): """ diff --git a/notebook/agentchat_pgvector_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat_pgvector.ipynb similarity index 100% rename from notebook/agentchat_pgvector_RetrieveChat.ipynb rename to notebook/agentchat_RetrieveChat_pgvector.ipynb diff --git a/notebook/agentchat_qdrant_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat_qdrant.ipynb similarity index 50% rename from notebook/agentchat_qdrant_RetrieveChat.ipynb rename to notebook/agentchat_RetrieveChat_qdrant.ipynb index 10d6b55e11c6..b5bc2f681d22 100644 --- a/notebook/agentchat_qdrant_RetrieveChat.ipynb +++ b/notebook/agentchat_RetrieveChat_qdrant.ipynb @@ -31,172 +31,19 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "Requirement already satisfied: pyautogen>=0.2.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (0.2.21)\n", - "Requirement already satisfied: flaml[automl] in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (2.1.2)\n", - "Requirement already satisfied: qdrant_client[fastembed] in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (1.6.4)\n", - "Requirement already satisfied: openai>=1.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.12.0)\n", - "Requirement already satisfied: diskcache in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (5.6.3)\n", - "Requirement already satisfied: termcolor in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2.3.0)\n", - "Requirement already satisfied: numpy<2,>=1.17.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.26.4)\n", - "Requirement already satisfied: python-dotenv in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.0.0)\n", - "Requirement already satisfied: tiktoken in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (0.5.1)\n", - "Requirement already satisfied: pydantic!=2.6.0,<3,>=1.10 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2.6.4)\n", - "Requirement already satisfied: docker in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (7.0.0)\n", - "Requirement already satisfied: lightgbm>=2.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (4.1.0)\n", - "Requirement already satisfied: xgboost>=0.90 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (2.0.1)\n", - "Requirement already satisfied: scipy>=1.4.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (1.10.1)\n", - "Requirement already satisfied: pandas>=1.1.4 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (2.2.0)\n", - "Requirement already satisfied: scikit-learn>=0.24 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (1.3.2)\n", - "Requirement already satisfied: grpcio>=1.41.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (1.60.0)\n", - "Requirement already satisfied: grpcio-tools>=1.41.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (1.59.2)\n", - "Requirement already satisfied: httpx>=0.14.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx[http2]>=0.14.0->qdrant_client[fastembed]) (0.25.1)\n", - "Requirement already satisfied: portalocker<3.0.0,>=2.7.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (2.8.2)\n", - "Requirement already satisfied: urllib3<2.0.0,>=1.26.14 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (1.26.18)\n", - "Requirement already satisfied: fastembed==0.1.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (0.1.1)\n", - "Requirement already satisfied: onnx<2.0,>=1.11 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (1.15.0)\n", - "Requirement already satisfied: onnxruntime<2.0,>=1.15 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (1.15.1)\n", - "Requirement already satisfied: requests<3.0,>=2.31 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (2.31.0)\n", - "Requirement already satisfied: tokenizers<0.14,>=0.13 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (0.13.3)\n", - "Requirement already satisfied: tqdm<5.0,>=4.65 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (4.66.2)\n", - "Requirement already satisfied: chromadb in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (0.4.22)\n", - "Requirement already satisfied: sentence-transformers in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (2.3.1)\n", - "Requirement already satisfied: pypdf in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (4.0.1)\n", - "Requirement already satisfied: ipython in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (8.17.2)\n", - "Requirement already satisfied: protobuf<5.0dev,>=4.21.6 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from grpcio-tools>=1.41.0->qdrant_client[fastembed]) (4.23.4)\n", - "Requirement already satisfied: setuptools in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from grpcio-tools>=1.41.0->qdrant_client[fastembed]) (68.2.2)\n", - "Requirement already satisfied: anyio in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (3.7.1)\n", - "Requirement already satisfied: certifi in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (2024.2.2)\n", - "Requirement already satisfied: httpcore in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (1.0.1)\n", - "Requirement already satisfied: idna in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (3.6)\n", - "Requirement already satisfied: sniffio in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (1.3.0)\n", - "Requirement already satisfied: h2<5,>=3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx[http2]>=0.14.0->qdrant_client[fastembed]) (4.1.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.8.0)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (4.9.0)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pandas>=1.1.4->flaml[automl]) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pandas>=1.1.4->flaml[automl]) (2024.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pandas>=1.1.4->flaml[automl]) (2024.1)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (0.6.0)\n", - "Requirement already satisfied: pydantic-core==2.16.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2.16.3)\n", - "Requirement already satisfied: joblib>=1.1.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from scikit-learn>=0.24->flaml[automl]) (1.3.2)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from scikit-learn>=0.24->flaml[automl]) (3.2.0)\n", - "Requirement already satisfied: build>=1.0.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.0.3)\n", - "Requirement already satisfied: chroma-hnswlib==0.7.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.7.3)\n", - "Requirement already satisfied: fastapi>=0.95.2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.104.1)\n", - "Requirement already satisfied: uvicorn>=0.18.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.24.0)\n", - "Requirement already satisfied: posthog>=2.4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (3.0.2)\n", - "Requirement already satisfied: pulsar-client>=3.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (3.3.0)\n", - "Requirement already satisfied: opentelemetry-api>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: pypika>=0.48.9 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.48.9)\n", - "Requirement already satisfied: overrides>=7.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (7.4.0)\n", - "Requirement already satisfied: importlib-resources in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (6.1.1)\n", - "Requirement already satisfied: bcrypt>=4.0.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (4.0.1)\n", - "Requirement already satisfied: typer>=0.9.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.9.0)\n", - "Requirement already satisfied: kubernetes>=28.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (28.1.0)\n", - "Requirement already satisfied: tenacity>=8.2.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (8.2.3)\n", - "Requirement already satisfied: PyYAML>=6.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (6.0.1)\n", - "Requirement already satisfied: mmh3>=4.0.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (4.0.1)\n", - "Requirement already satisfied: packaging>=14.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from docker->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (23.2)\n", - "Requirement already satisfied: decorator in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (0.19.1)\n", - "Requirement already satisfied: matplotlib-inline in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (0.1.6)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (3.0.39)\n", - "Requirement already satisfied: pygments>=2.4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (2.16.1)\n", - "Requirement already satisfied: stack-data in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (0.6.3)\n", - "Requirement already satisfied: traitlets>=5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (5.14.2)\n", - "Requirement already satisfied: exceptiongroup in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (1.1.3)\n", - "Requirement already satisfied: pexpect>4.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (4.8.0)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.32.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (4.33.3)\n", - "Requirement already satisfied: torch>=1.11.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.2.0)\n", - "Requirement already satisfied: nltk in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.15.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (0.20.3)\n", - "Requirement already satisfied: Pillow in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (10.2.0)\n", - "Requirement already satisfied: regex>=2022.1.18 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from tiktoken->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2023.12.25)\n", - "Requirement already satisfied: pyproject_hooks in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from build>=1.0.3->chromadb->pyautogen[retrievechat]>=0.2.3) (1.0.0)\n", - "Requirement already satisfied: tomli>=1.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from build>=1.0.3->chromadb->pyautogen[retrievechat]>=0.2.3) (2.0.1)\n", - "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb->pyautogen[retrievechat]>=0.2.3) (0.27.0)\n", - "Requirement already satisfied: hyperframe<7,>=6.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from h2<5,>=3->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (6.0.1)\n", - "Requirement already satisfied: hpack<5,>=4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from h2<5,>=3->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (4.0.0)\n", - "Requirement already satisfied: filelock in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from huggingface-hub>=0.15.1->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.13.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from huggingface-hub>=0.15.1->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2024.2.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from jedi>=0.16->ipython->pyautogen[retrievechat]>=0.2.3) (0.8.3)\n", - "Requirement already satisfied: six>=1.9.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.16.0)\n", - "Requirement already satisfied: google-auth>=1.0.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (2.23.4)\n", - "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.6.4)\n", - "Requirement already satisfied: requests-oauthlib in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.3.1)\n", - "Requirement already satisfied: oauthlib>=3.2.2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (3.2.2)\n", - "Requirement already satisfied: coloredlogs in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (15.0.1)\n", - "Requirement already satisfied: flatbuffers in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (23.5.26)\n", - "Requirement already satisfied: sympy in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (1.12)\n", - "Requirement already satisfied: deprecated>=1.2.6 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.2.14)\n", - "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (6.11.0)\n", - "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (2.2.1)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.61.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.20.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.20.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-instrumentation==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-util-http==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.41b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.16.0)\n", - "Requirement already satisfied: asgiref~=3.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-asgi==0.41b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (3.7.2)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pexpect>4.3->ipython->pyautogen[retrievechat]>=0.2.3) (0.7.0)\n", - "Requirement already satisfied: monotonic>=1.5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.6)\n", - "Requirement already satisfied: wcwidth in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython->pyautogen[retrievechat]>=0.2.3) (0.2.9)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from requests<3.0,>=2.31->fastembed==0.1.1->qdrant_client[fastembed]) (3.3.2)\n", - "Requirement already satisfied: networkx in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.1.3)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (8.9.2.26)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.19.3)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: triton==2.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.2.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.3.52)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from transformers<5.0.0,>=4.32.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (0.3.2)\n", - "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from typer>=0.9.0->chromadb->pyautogen[retrievechat]>=0.2.3) (8.1.7)\n", - "Requirement already satisfied: h11>=0.8 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn>=0.18.3->uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.14.0)\n", - "Requirement already satisfied: httptools>=0.5.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.6.1)\n", - "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.19.0)\n", - "Requirement already satisfied: watchfiles>=0.13 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.21.0)\n", - "Requirement already satisfied: websockets>=10.4 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (11.0.3)\n", - "Requirement already satisfied: executing>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from stack-data->ipython->pyautogen[retrievechat]>=0.2.3) (2.0.1)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from stack-data->ipython->pyautogen[retrievechat]>=0.2.3) (2.4.1)\n", - "Requirement already satisfied: pure-eval in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from stack-data->ipython->pyautogen[retrievechat]>=0.2.3) (0.2.2)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (5.3.2)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.3.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (4.9)\n", - "Requirement already satisfied: zipp>=0.5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (3.17.0)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from coloredlogs->onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (10.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from jinja2->torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.1.5)\n", - "Requirement already satisfied: mpmath>=0.19 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sympy->onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (1.3.0)\n", - "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.5.0)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ - "%pip install \"pyautogen[retrievechat-qdrant]\" \"flaml[automl]\"" + "%pip install \"pyautogen[retrievechat-qdrant]\" \"flaml[automl]\" -q" ] }, { @@ -211,23 +58,24 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "models to use: ['gpt4-1106-preview', 'gpt-35-turbo', 'gpt-35-turbo-0613']\n" + "models to use: ['gpt4-1106-preview', 'gpt-4o', 'gpt-35-turbo', 'gpt-35-turbo-0613']\n" ] } ], "source": [ "from qdrant_client import QdrantClient\n", + "from sentence_transformers import SentenceTransformer\n", "\n", "import autogen\n", - "from autogen.agentchat.contrib.qdrant_retrieve_user_proxy_agent import QdrantRetrieveUserProxyAgent\n", "from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent\n", + "from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent\n", "\n", "# Accepted file formats for that can be stored in\n", "# a vector database instance\n", @@ -253,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -261,7 +109,7 @@ "output_type": "stream", "text": [ "Accepted file formats for `docs_path`:\n", - "['yml', 'ppt', 'org', 'doc', 'epub', 'rst', 'log', 'docx', 'htm', 'html', 'tsv', 'csv', 'json', 'yaml', 'xlsx', 'pptx', 'rtf', 'msg', 'odt', 'pdf', 'jsonl', 'md', 'xml', 'txt']\n" + "['rtf', 'jsonl', 'xml', 'json', 'md', 'rst', 'docx', 'msg', 'pdf', 'log', 'xlsx', 'org', 'txt', 'csv', 'pptx', 'tsv', 'yml', 'epub', 'yaml', 'ppt', 'htm', 'doc', 'odt', 'html']\n" ] } ], @@ -284,9 +132,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "67171b10626248ba8b5bff0f5a4d6895", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 5 files: 0%| | 0/5 [00:003 Agents) diff --git a/website/docs/ecosystem/pgvector.md b/website/docs/ecosystem/pgvector.md index c455595eda4e..99afa676e7e4 100644 --- a/website/docs/ecosystem/pgvector.md +++ b/website/docs/ecosystem/pgvector.md @@ -2,4 +2,4 @@ [PGVector](https://github.com/pgvector/pgvector) is an open-source vector similarity search for Postgres. -- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_pgvector_RetrieveChat.ipynb) +- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) diff --git a/website/docs/installation/Optional-Dependencies.md b/website/docs/installation/Optional-Dependencies.md index 13991023f81b..2d0067c9950e 100644 --- a/website/docs/installation/Optional-Dependencies.md +++ b/website/docs/installation/Optional-Dependencies.md @@ -75,7 +75,7 @@ Example notebooks: [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) -[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) +[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) ## Teachability diff --git a/website/docs/topics/retrieval_augmentation.md b/website/docs/topics/retrieval_augmentation.md index ca7222a404f3..366893cb9825 100644 --- a/website/docs/topics/retrieval_augmentation.md +++ b/website/docs/topics/retrieval_augmentation.md @@ -119,8 +119,8 @@ ragproxyagent.initiate_chat( ## More Examples and Notebooks For more detailed examples and notebooks showcasing the usage of retrieval augmented agents in AutoGen, refer to the following: - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_RetrieveChat) -- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_pgvector_RetrieveChat.ipynb) -- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) +- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) - Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_RAG) From 5e937641cd242000d83776943970168d0943670c Mon Sep 17 00:00:00 2001 From: Luca Date: Wed, 17 Jul 2024 14:17:08 +0200 Subject: [PATCH 45/57] Fixed a typo in tool-use.ipynb (#3151) Fixed a typo in tool-use.ipynb: comaptible -> compatible --- website/docs/tutorial/tool-use.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/tutorial/tool-use.ipynb b/website/docs/tutorial/tool-use.ipynb index b5e21559225c..792bfefde878 100644 --- a/website/docs/tutorial/tool-use.ipynb +++ b/website/docs/tutorial/tool-use.ipynb @@ -21,7 +21,7 @@ "````mdx-code-block\n", ":::note\n", "Tool use is currently only available for LLMs\n", - "that support OpenAI-comaptible tool call API.\n", + "that support OpenAI-compatible tool call API.\n", ":::\n", "````" ] From 85b83e7af6d6793d1d97f9e831d4968bccff4c51 Mon Sep 17 00:00:00 2001 From: Hugh Lyu Date: Thu, 18 Jul 2024 09:25:29 +0800 Subject: [PATCH 46/57] Add Agentok into gallery (#3148) * docs: Added Agentok into gallery. * Fixed the format issue * Track agentok.png with Git LFS --------- Co-authored-by: Qingyun Wu --- website/src/data/gallery.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/website/src/data/gallery.json b/website/src/data/gallery.json index cf68764e241c..10ed9f6866db 100644 --- a/website/src/data/gallery.json +++ b/website/src/data/gallery.json @@ -219,5 +219,14 @@ "description": "Automatically creating linear tasks and assigning them to the right person, project, and team from GitHub commits using AutoGen Agents.", "image": "composio-autogen.png", "tags": ["tools"] + }, + { + "title": "Agentok - Autogen Visualized", + "link": "https://github.com/hughlv/agentok", + "description": "Offering intuitive visual tools that streamline the creation and management of complex AutoGen workflows.", + "image": "agentok.png", + "tags": [ + "tools", "ui", "app" + ] } ] From 0beb0a7f616d5194c03329af4ae45f27304b2a4a Mon Sep 17 00:00:00 2001 From: Nikita Fedyashev Date: Thu, 18 Jul 2024 04:27:01 +0300 Subject: [PATCH 47/57] Fix typo in agentchat_nestedchat.ipynb (#3139) --- notebook/agentchat_nestedchat.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_nestedchat.ipynb b/notebook/agentchat_nestedchat.ipynb index 3cd4d0a99ed7..f81f20398591 100644 --- a/notebook/agentchat_nestedchat.ipynb +++ b/notebook/agentchat_nestedchat.ipynb @@ -100,7 +100,7 @@ " system_message=\"\"\"\n", " You are a professional writer, known for your insightful and engaging articles.\n", " You transform complex concepts into compelling narratives.\n", - " You should imporve the quality of the content based on the feedback from the user.\n", + " You should improve the quality of the content based on the feedback from the user.\n", " \"\"\",\n", ")\n", "\n", From 0cdbc345c56aa0708eb4a15ee70d87b17b5dc0d4 Mon Sep 17 00:00:00 2001 From: Umer Mansoor Date: Wed, 17 Jul 2024 19:07:40 -0700 Subject: [PATCH 48/57] Update JSON_mode_example.ipynb (#3130) Improve minor mistakes in documentation --- notebook/JSON_mode_example.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebook/JSON_mode_example.ipynb b/notebook/JSON_mode_example.ipynb index f3adce9dec35..c4b65c4d9f4d 100644 --- a/notebook/JSON_mode_example.ipynb +++ b/notebook/JSON_mode_example.ipynb @@ -26,7 +26,7 @@ "\n", "\n", "## Requirements\n", - "JSON mode is a feature of OpenAI API, however strong models (such as claude 3 Opus), can generate appropriate json as well.\n", + "JSON mode is a feature of OpenAI API, however strong models (such as Claude 3 Opus), can generate appropriate json as well.\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "```bash\n", "pip install pyautogen\n", @@ -71,7 +71,7 @@ "source": [ "## Model Configuration\n", "\n", - "we Need to set two different Configs for this to work. \n", + "We need to set two different Configs for this to work. \n", "One for JSON mode\n", "One for Text mode. \n", "This is because the group chat manager requires text mode. " From bd735d0f825b5ea479f05d34c82cb68a7b421f3b Mon Sep 17 00:00:00 2001 From: Li Jiang Date: Fri, 19 Jul 2024 19:50:34 +0800 Subject: [PATCH 49/57] Fix docstring (#3172) --- autogen/agentchat/contrib/retrieve_user_proxy_agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py index 1471afb07eaf..4842bd4e9f53 100644 --- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py +++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py @@ -136,7 +136,7 @@ def __init__( - `client` (Optional, chromadb.Client) - the chromadb client. If key not provided, a default client `chromadb.Client()` will be used. If you want to use other vector db, extend this class and override the `retrieve_docs` function. - **Deprecated**: use `vector_db` instead. + *[Deprecated]* use `vector_db` instead. - `docs_path` (Optional, Union[str, List[str]]) - the path to the docs directory. It can also be the path to a single file, the url to a single file or a list of directories, files and urls. Default is None, which works only if the @@ -150,7 +150,7 @@ def __init__( By default, "extra_docs" is set to false, starting document IDs from zero. This poses a risk as new documents might overwrite existing ones, potentially causing unintended loss or alteration of data in the collection. - **Deprecated**: use `new_docs` when use `vector_db` instead of `client`. + *[Deprecated]* use `new_docs` when use `vector_db` instead of `client`. - `new_docs` (Optional, bool) - when True, only adds new documents to the collection; when False, updates existing documents and adds new ones. Default is True. Document id is used to determine if a document is new or existing. By default, the @@ -173,7 +173,7 @@ def __init__( models can be found at `https://www.sbert.net/docs/pretrained_models.html`. The default model is a fast model. If you want to use a high performance model, `all-mpnet-base-v2` is recommended. - **Deprecated**: no need when use `vector_db` instead of `client`. + *[Deprecated]* no need when use `vector_db` instead of `client`. - `embedding_function` (Optional, Callable) - the embedding function for creating the vector db. Default is None, SentenceTransformer with the given `embedding_model` will be used. If you want to use OpenAI, Cohere, HuggingFace or other embedding @@ -220,7 +220,7 @@ def __init__( Example of overriding retrieve_docs - If you have set up a customized vector db, and it's not compatible with chromadb, you can easily plug in it with below code. - **Deprecated**: Use `vector_db` instead. You can extend VectorDB and pass it to the agent. + *[Deprecated]* use `vector_db` instead. You can extend VectorDB and pass it to the agent. ```python class MyRetrieveUserProxyAgent(RetrieveUserProxyAgent): def query_vector_db( From d07367815404ae37754b637231e3b095f802f124 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Fri, 19 Jul 2024 09:36:04 -0700 Subject: [PATCH 50/57] add streaming tool call example (#3167) --- .../GettingStart/Streaming_Tool_Call.cs | 56 +++++++++++++++++++ .../Orchestrator/RolePlayOrchestratorTests.cs | 31 ++++++++++ 2 files changed, 87 insertions(+) create mode 100644 dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs new file mode 100644 index 000000000000..48ebd127b562 --- /dev/null +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs @@ -0,0 +1,56 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Streaming_Tool_Call.cs + +using AutoGen.Core; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using FluentAssertions; + +namespace AutoGen.BasicSample.GettingStart; + +internal class Streaming_Tool_Call +{ + public static async Task RunAsync() + { + #region Create_tools + var tools = new Tools(); + #endregion Create_tools + + #region Create_auto_invoke_middleware + var autoInvokeMiddleware = new FunctionCallMiddleware( + functions: [tools.GetWeatherFunctionContract], + functionMap: new Dictionary>>() + { + { tools.GetWeatherFunctionContract.Name, tools.GetWeatherWrapper }, + }); + #endregion Create_auto_invoke_middleware + + #region Create_Agent + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); + var model = "gpt-4o"; + var openaiClient = new OpenAIClient(apiKey); + var agent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "agent", + modelName: model, + systemMessage: "You are a helpful AI assistant") + .RegisterMessageConnector() + .RegisterStreamingMiddleware(autoInvokeMiddleware) + .RegisterPrintMessage(); + #endregion Create_Agent + + IMessage finalReply = null; + var question = new TextMessage(Role.User, "What's the weather in Seattle"); + + // In streaming function call + // function can only be invoked untill all the chunks are collected + // therefore, only one ToolCallAggregateMessage chunk will be return here. + await foreach (var message in agent.GenerateStreamingReplyAsync([question])) + { + finalReply = message; + } + + finalReply?.GetContent().Should().Be("The weather in Seattle is sunny."); + } +} diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs index eea0a90c1caf..5a2cebb66cff 100644 --- a/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs +++ b/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs @@ -228,6 +228,37 @@ public async Task GPT_3_5_CoderReviewerRunnerTestAsync() await CoderReviewerRunnerTestAsync(openAIChatAgent); } + [ApiKeyFact("OPENAI_API_KEY")] + public async Task GPT_4o_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); + var model = "gpt-4o"; + var openaiClient = new OpenAIClient(apiKey); + var openAIChatAgent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "assistant", + modelName: model) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(openAIChatAgent); + } + + [ApiKeyFact("OPENAI_API_KEY")] + public async Task GPT_4o_mini_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); + var model = "gpt-4o-mini"; + var openaiClient = new OpenAIClient(apiKey); + var openAIChatAgent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "assistant", + modelName: model) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(openAIChatAgent); + } + + [ApiKeyFact("GOOGLE_GEMINI_API_KEY")] public async Task GoogleGemini_1_5_flash_001_CoderReviewerRunnerTestAsync() { From 1fdeca4205e0d85282c7f9fd8beee3de377345a7 Mon Sep 17 00:00:00 2001 From: Manojkumar Kotakonda <44414430+makkzone@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:43:09 -0300 Subject: [PATCH 51/57] Added anthropic bedrock (#3103) * Added anthropic bedrock * Code format and fixed import * Added tests for anthropic bedrock * tests update --------- Co-authored-by: Chi Wang Co-authored-by: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> --- autogen/oai/anthropic.py | 75 ++++++++++++++++++++++++++++++++++++-- test/oai/test_anthropic.py | 32 +++++++++++++++- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/autogen/oai/anthropic.py b/autogen/oai/anthropic.py index e2448929e618..62078d42631d 100644 --- a/autogen/oai/anthropic.py +++ b/autogen/oai/anthropic.py @@ -16,6 +16,27 @@ ] assistant = autogen.AssistantAgent("assistant", llm_config={"config_list": config_list}) + +Example usage for Anthropic Bedrock: + +Install the `anthropic` package by running `pip install --upgrade anthropic`. +- https://docs.anthropic.com/en/docs/quickstart-guide + +import autogen + +config_list = [ + { + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_access_key":, + "aws_secret_key":, + "aws_session_token":, + "aws_region":"us-east-1", + "api_type": "anthropic", + } +] + +assistant = autogen.AssistantAgent("assistant", llm_config={"config_list": config_list}) + """ from __future__ import annotations @@ -28,7 +49,7 @@ import warnings from typing import Any, Dict, List, Tuple, Union -from anthropic import Anthropic +from anthropic import Anthropic, AnthropicBedrock from anthropic import __version__ as anthropic_version from anthropic.types import Completion, Message, TextBlock, ToolUseBlock from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall @@ -64,14 +85,44 @@ def __init__(self, **kwargs: Any): api_key (str): The API key for the Anthropic API or set the `ANTHROPIC_API_KEY` environment variable. """ self._api_key = kwargs.get("api_key", None) + self._aws_access_key = kwargs.get("aws_access_key", None) + self._aws_secret_key = kwargs.get("aws_secret_key", None) + self._aws_session_token = kwargs.get("aws_session_token", None) + self._aws_region = kwargs.get("aws_region", None) if not self._api_key: self._api_key = os.getenv("ANTHROPIC_API_KEY") - if self._api_key is None: - raise ValueError("API key is required to use the Anthropic API.") + if not self._aws_access_key: + self._aws_access_key = os.getenv("AWS_ACCESS_KEY") + + if not self._aws_secret_key: + self._aws_secret_key = os.getenv("AWS_SECRET_KEY") + + if not self._aws_session_token: + self._aws_session_token = os.getenv("AWS_SESSION_TOKEN") + + if not self._aws_region: + self._aws_region = os.getenv("AWS_REGION") + + if self._api_key is None and ( + self._aws_access_key is None + or self._aws_secret_key is None + or self._aws_session_token is None + or self._aws_region is None + ): + raise ValueError("API key or AWS credentials are required to use the Anthropic API.") + + if self._api_key is not None: + self._client = Anthropic(api_key=self._api_key) + else: + self._client = AnthropicBedrock( + aws_access_key=self._aws_access_key, + aws_secret_key=self._aws_secret_key, + aws_session_token=self._aws_session_token, + aws_region=self._aws_region, + ) - self._client = Anthropic(api_key=self._api_key) self._last_tooluse_status = {} def load_config(self, params: Dict[str, Any]): @@ -107,6 +158,22 @@ def cost(self, response) -> float: def api_key(self): return self._api_key + @property + def aws_access_key(self): + return self._aws_access_key + + @property + def aws_secret_key(self): + return self._aws_secret_key + + @property + def aws_session_token(self): + return self._aws_session_token + + @property + def aws_region(self): + return self._aws_region + def create(self, params: Dict[str, Any]) -> Completion: if "tools" in params: converted_functions = self.convert_tools_to_functions(params["tools"]) diff --git a/test/oai/test_anthropic.py b/test/oai/test_anthropic.py index 379ab47f6756..53926dbd18d6 100644 --- a/test/oai/test_anthropic.py +++ b/test/oai/test_anthropic.py @@ -50,17 +50,47 @@ def anthropic_client(): @pytest.mark.skipif(skip, reason=reason) def test_initialization_missing_api_key(): os.environ.pop("ANTHROPIC_API_KEY", None) - with pytest.raises(ValueError, match="API key is required to use the Anthropic API."): + os.environ.pop("AWS_ACCESS_KEY", None) + os.environ.pop("AWS_SECRET_KEY", None) + os.environ.pop("AWS_SESSION_TOKEN", None) + os.environ.pop("AWS_REGION", None) + with pytest.raises(ValueError, match="API key or AWS credentials are required to use the Anthropic API."): AnthropicClient() AnthropicClient(api_key="dummy_api_key") +@pytest.fixture() +def anthropic_client_with_aws_credentials(): + return AnthropicClient( + aws_access_key="dummy_access_key", + aws_secret_key="dummy_secret_key", + aws_session_token="dummy_session_token", + aws_region="us-west-2", + ) + + @pytest.mark.skipif(skip, reason=reason) def test_intialization(anthropic_client): assert anthropic_client.api_key == "dummy_api_key", "`api_key` should be correctly set in the config" +@pytest.mark.skipif(skip, reason=reason) +def test_intialization_with_aws_credentials(anthropic_client_with_aws_credentials): + assert ( + anthropic_client_with_aws_credentials.aws_access_key == "dummy_access_key" + ), "`aws_access_key` should be correctly set in the config" + assert ( + anthropic_client_with_aws_credentials.aws_secret_key == "dummy_secret_key" + ), "`aws_secret_key` should be correctly set in the config" + assert ( + anthropic_client_with_aws_credentials.aws_session_token == "dummy_session_token" + ), "`aws_session_token` should be correctly set in the config" + assert ( + anthropic_client_with_aws_credentials.aws_region == "us-west-2" + ), "`aws_region` should be correctly set in the config" + + # Test cost calculation @pytest.mark.skipif(skip, reason=reason) def test_cost_calculation(mock_completion): From d1d4b10479b8b6170693cd15e9f3d1cf2484b6b6 Mon Sep 17 00:00:00 2001 From: Umer Mansoor Date: Fri, 19 Jul 2024 13:17:18 -0700 Subject: [PATCH 52/57] Update token_count_utils.py - Replace `print` with `logger.warning` for consistency (#3168) The code was using both `logger.warning` and `print` for showing warning. This commit fixes this inconsistency which can be an issue on production environments / logging systems --- autogen/token_count_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autogen/token_count_utils.py b/autogen/token_count_utils.py index 2842a7494536..365285e09551 100644 --- a/autogen/token_count_utils.py +++ b/autogen/token_count_utils.py @@ -95,7 +95,7 @@ def _num_token_from_messages(messages: Union[List, Dict], model="gpt-3.5-turbo-0 try: encoding = tiktoken.encoding_for_model(model) except KeyError: - print("Warning: model not found. Using cl100k_base encoding.") + logger.warning(f"Model {model} not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base") if model in { "gpt-3.5-turbo-0613", @@ -166,7 +166,7 @@ def num_tokens_from_functions(functions, model="gpt-3.5-turbo-0613") -> int: try: encoding = tiktoken.encoding_for_model(model) except KeyError: - print("Warning: model not found. Using cl100k_base encoding.") + logger.warning(f"Model {model} not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base") num_tokens = 0 @@ -193,7 +193,7 @@ def num_tokens_from_functions(functions, model="gpt-3.5-turbo-0613") -> int: function_tokens += 3 function_tokens += len(encoding.encode(o)) else: - print(f"Warning: not supported field {field}") + logger.warning(f"Not supported field {field}") function_tokens += 11 if len(parameters["properties"]) == 0: function_tokens -= 2 From 2887e0757e52c31d5ad187d896bddce484a16270 Mon Sep 17 00:00:00 2001 From: Sugato Ray Date: Sun, 21 Jul 2024 01:12:15 -0400 Subject: [PATCH 53/57] fix: update method name in GeminiClient (#3007) - change from `_initialize_vartexai` to `_initialize_vertexai` Co-authored-by: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> --- autogen/oai/gemini.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autogen/oai/gemini.py b/autogen/oai/gemini.py index 8babb8727e3c..73d41cddbf53 100644 --- a/autogen/oai/gemini.py +++ b/autogen/oai/gemini.py @@ -72,7 +72,7 @@ class GeminiClient: "max_output_tokens": "max_output_tokens", } - def _initialize_vartexai(self, **params): + def _initialize_vertexai(self, **params): if "google_application_credentials" in params: # Path to JSON Keyfile os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = params["google_application_credentials"] @@ -106,7 +106,7 @@ def __init__(self, **kwargs): self.api_key = os.getenv("GOOGLE_API_KEY") if self.api_key is None: self.use_vertexai = True - self._initialize_vartexai(**kwargs) + self._initialize_vertexai(**kwargs) else: self.use_vertexai = False else: @@ -142,7 +142,7 @@ def get_usage(response) -> Dict: def create(self, params: Dict) -> ChatCompletion: if self.use_vertexai: - self._initialize_vartexai(**params) + self._initialize_vertexai(**params) else: assert ("project_id" not in params) and ( "location" not in params From adf438abfa32f8bf4bab9298cb3a416b34d14ef2 Mon Sep 17 00:00:00 2001 From: Olaoluwa Ademola Salami Date: Sun, 21 Jul 2024 19:42:26 +0100 Subject: [PATCH 54/57] add Use AutoGen.Net agent as model in AG Studio (#3182) * add Use AutoGen.Net agent as model in AG Studio * add git lfs * test * dotnet/nuget/icon.png,dotnet/resource/images/background.png,dotnet/resource/images/square.png,dotnet/test/AutoGen.Anthropic.Tests/images/square.png,dotnet/test/AutoGen.Ollama.Tests/images/image.png,dotnet/test/AutoGen.Ollama.Tests/images/square.png,dotnet/test/AutoGen.Tests/ApprovalTests/square.png,dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png,dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png,dotnet/website/images/articles/CreateUserProxyAgent/image-1.png,dotnet/website/images/articles/PrintMessageMiddleware/printMessage.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png,dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png,notebook/friendly_and_suspicous.jpg,notebook/nested-chats-chess.png,notebook/nested_chat_1.png,notebook/nested_chat_2.png,notebook/optiGuide_new_design.png,notebook/viz_gc.png,samples/apps/auto-anny/images/icon.png,samples/apps/autogen-studio/docs/ara_stockprices.png,samples/apps/autogen-studio/frontend/src/images/icon.png,test/test_files/test_image.png,website/blog/2023-04-21-LLM-tuning-math/img/level2algebra.png,website/blog/2023-04-21-LLM-tuning-math/img/level3algebra.png,website/blog/2023-04-21-LLM-tuning-math/img/level4algebra.png,website/blog/2023-04-21-LLM-tuning-math/img/level5algebra.png,website/blog/2023-05-18-GPT-adaptive-humaneval/img/design.png,website/blog/2023-05-18-GPT-adaptive-humaneval/img/humaneval.png,website/blog/2023-06-28-MathChat/img/mathchatflow.png,website/blog/2023-06-28-MathChat/img/result.png,website/blog/2023-10-18-RetrieveChat/img/retrievechat-arch.png,website/blog/2023-10-26-TeachableAgent/img/teachable-arch.png,website/blog/2023-11-06-LMM-Agent/img/teaser.png,website/blog/2023-11-09-EcoAssistant/img/chat.png,website/blog/2023-11-09-EcoAssistant/img/results.png,website/blog/2023-11-09-EcoAssistant/img/system.png,website/blog/2023-11-09-EcoAssistant/img/template-demo.png,website/blog/2023-11-09-EcoAssistant/img/template.png,website/blog/2023-11-13-OAI-assistants/img/teaser.jpg,website/blog/2023-11-20-AgentEval/img/agenteval-CQ.png,website/blog/2023-11-20-AgentEval/img/math-problems-plot.png,website/blog/2023-11-20-AgentEval/img/tasks-taxonomy.png,website/blog/2023-11-26-Agent-AutoBuild/img/agent_autobuild.png,website/blog/2023-12-01-AutoGenStudio/img/autogenstudio_config.png,website/blog/2023-12-01-AutoGenStudio/img/autogenstudio_home.png,website/blog/2023-12-01-AutoGenStudio/img/autogenstudio_skills.png,website/blog/2023-12-23-AgentOptimizer/img/agentoptimizer.png,website/blog/2024-01-25-AutoGenBench/img/teaser.jpg,website/blog/2024-02-02-AutoAnny/img/AutoAnnyLogo.jpg,website/blog/2024-02-11-FSM-GroupChat/img/FSM_logic.png,website/blog/2024-02-11-FSM-GroupChat/img/FSM_of_multi-agents.png,website/blog/2024-02-11-FSM-GroupChat/img/teaser.jpg,website/blog/2024-02-29-StateFlow/img/alfworld.png,website/blog/2024-02-29-StateFlow/img/bash_result.png,website/blog/2024-02-29-StateFlow/img/intercode.png,website/blog/2024-02-29-StateFlow/img/sf_example_1.png,website/blog/2024-03-03-AutoGen-Update/img/contributors.png,website/blog/2024-03-03-AutoGen-Update/img/dalle_gpt4v.png,website/blog/2024-03-03-AutoGen-Update/img/gaia.png,website/blog/2024-03-03-AutoGen-Update/img/love.png,website/blog/2024-03-03-AutoGen-Update/img/teach.png,website/blog/2024-03-11-AutoDefense/imgs/architecture.png,website/blog/2024-03-11-AutoDefense/imgs/defense-agency-design.png,website/blog/2024-03-11-AutoDefense/imgs/table-4agents.png,website/blog/2024-03-11-AutoDefense/imgs/table-agents.png,website/blog/2024-03-11-AutoDefense/imgs/table-compared-methods.png,website/blog/2024-05-24-Agent/img/agents.png,website/blog/2024-05-24-Agent/img/leadership.png,website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png,website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg,website/docs/Use-Cases/images/agent_example.png,website/docs/Use-Cases/images/app.png,website/docs/Use-Cases/images/autogen_agents.png,website/docs/autogen-studio/img/agent_assistant.png,website/docs/autogen-studio/img/agent_groupchat.png,website/docs/autogen-studio/img/agent_new.png,website/docs/autogen-studio/img/agent_skillsmodel.png,website/docs/autogen-studio/img/ara_stockprices.png,website/docs/autogen-studio/img/model_new.png,website/docs/autogen-studio/img/model_openai.png,website/docs/autogen-studio/img/skill.png,website/docs/autogen-studio/img/workflow_chat.png,website/docs/autogen-studio/img/workflow_export.png,website/docs/autogen-studio/img/workflow_new.png,website/docs/autogen-studio/img/workflow_profile.png,website/docs/autogen-studio/img/workflow_sequential.png,website/docs/autogen-studio/img/workflow_test.png,website/docs/ecosystem/img/ecosystem-composio.png,website/docs/ecosystem/img/ecosystem-databricks.png,website/docs/ecosystem/img/ecosystem-fabric.png,website/docs/ecosystem/img/ecosystem-llamaindex.png,website/docs/ecosystem/img/ecosystem-memgpt.png,website/docs/ecosystem/img/ecosystem-ollama.png,website/docs/ecosystem/img/ecosystem-promptflow.png,website/docs/topics/non-openai-models/images/cloudlocalproxy.png,website/docs/tutorial/assets/code-execution-in-conversation.png,website/docs/tutorial/assets/code-executor-docker.png,website/docs/tutorial/assets/code-executor-no-docker.png,website/docs/tutorial/assets/conversable-agent.jpg,website/docs/tutorial/assets/group-chat.png,website/docs/tutorial/assets/human-in-the-loop.png,website/docs/tutorial/assets/nested-chats.png,website/docs/tutorial/assets/sequential-two-agent-chat.png,website/docs/tutorial/assets/two-agent-chat.png,website/static/img/autogen_agentchat.png,website/static/img/autogen_app.png,website/static/img/chat_example.png,website/static/img/create_gcp_svc.png,website/static/img/gallery/TensionCode.png,website/static/img/gallery/autotx.png,website/static/img/gallery/composio-autogen.png,website/static/img/gallery/default.png,website/static/img/gallery/robot.jpg,website/static/img/gallery/webagent.jpg,website/static/img/gallery/x-force-ide-ui.png: convert to Git LFS --- .../FinalStepsA.png | 3 + .../FinalStepsB.png | 3 + .../FinalStepsC.png | 3 + .../Step5.2OpenAIModel.png | 3 + .../Step5.3ModelNameAndURL.png | 3 + .../UseAutoGenAsModelinAGStudio/Step6.png | 3 + .../UseAutoGenAsModelinAGStudio/Step6b.png | 3 + .../UseAutoGenAsModelinAGStudio/Terminal.png | 3 + .../TheModelTab.png | 3 + ...AutoGen.Net-agent-as-model-in-AG-Studio.md | 84 +++++++++++++++++++ dotnet/website/tutorial/toc.yml | 5 +- 11 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png create mode 100644 dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png create mode 100644 dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png new file mode 100644 index 000000000000..0403a8cf9742 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:491f8f538c55ce8768179cabfd3789c71c4a07b7d809f85deba9b8f4b759c00e +size 42329 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png new file mode 100644 index 000000000000..03a68735c082 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e319fad11682c46c3dc511e2fc63e033f3f99efb06d4530e7f72d1f4af23848f +size 31528 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png new file mode 100644 index 000000000000..7326ad14d040 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8024b5336615e8c2c3497df7a5890a331bd5bdc7b15dd06abd7ec528ffe0932 +size 70169 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png new file mode 100644 index 000000000000..b2b7481bbe78 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:911f2f7c1ab4f9403386298d9769243c0aa8cc22c6f119342cc107a654d1463a +size 44041 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png new file mode 100644 index 000000000000..d1c19f300806 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec10a48ed3f0a6d8448e0ce425658f3857c2cf89e2badef8a8d3a8c3744fc3bf +size 51944 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png new file mode 100644 index 000000000000..67c734454427 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f016faea51f64af3970fde41ac95249c4e0423b02573f058c36dc1e6ba15562d +size 50669 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png new file mode 100644 index 000000000000..ebd19bff045a --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a23cbbf5d3d24eaf1da9370e0914f186815f2ecbf46131d2fd6eb5ff3264d96 +size 22569 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png new file mode 100644 index 000000000000..9edefc3aebf3 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97328776c25fd0a61c76065db379406d8d3c96bd8773490c34c168cd7c69a855 +size 58527 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png new file mode 100644 index 000000000000..55e7bd862613 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d7f4f3a772278e6de320a3601a76f8a9862cab4a9c0da03fad3058b86fcfaf7 +size 45260 diff --git a/dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md b/dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md new file mode 100644 index 000000000000..a47cb01f649e --- /dev/null +++ b/dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md @@ -0,0 +1,84 @@ +This tutorial shows how to use AutoGen.Net agent as model in AG Studio + +## Step 1. Create Dotnet empty web app and install AutoGen and AutoGen.WebAPI package + +```bash +dotnet new web +dotnet add package AutoGen +dotnet add package AutoGen.WebAPI +``` + +## Step 2. Replace the Program.cs with following code + +```bash +using AutoGen.Core; +using AutoGen.Service; + +var builder = WebApplication.CreateBuilder(args); +var app = builder.Build(); + +var helloWorldAgent = new HelloWorldAgent(); +app.UseAgentAsOpenAIChatCompletionEndpoint(helloWorldAgent); + +app.Run(); + +class HelloWorldAgent : IAgent +{ + public string Name => "HelloWorld"; + + public Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(new TextMessage(Role.Assistant, "Hello World!", from: this.Name)); + } +} +``` + +## Step 3: Start the web app + +Run the following command to start web api + +```bash +dotnet RUN +``` + +The web api will listen at `http://localhost:5264/v1/chat/completion + +![terminal](../images/articles/UseAutoGenAsModelinAGStudio/Terminal.png) + +## Step 4: In another terminal, start autogen-studio + +```bash +autogenstudio ui +``` + +## Step 5: Navigate to AutoGen Studio UI and add hello world agent as openai Model + +### Step 5.1: Go to model tab + +![The Model Tab](../images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png) + +### Step 5.2: Select "OpenAI model" card + +![Open AI model Card](../images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png) + +### Step 5.3: Fill the model name and url + +The model name needs to be same with agent name + +![Fill the model name and url](../images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png) + +## Step 6: Create a hello world agent that uses the hello world model + +![Create a hello world agent that uses the hello world model](../images/articles/UseAutoGenAsModelinAGStudio/Step6.png) + +![Agent Configuration](../images/articles/UseAutoGenAsModelinAGStudio/Step6b.png) + +## Final Step: Use the hello world agent in workflow + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png) + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png) + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png) + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png) diff --git a/dotnet/website/tutorial/toc.yml b/dotnet/website/tutorial/toc.yml index f624ec4af289..167baa70e4fd 100644 --- a/dotnet/website/tutorial/toc.yml +++ b/dotnet/website/tutorial/toc.yml @@ -5,4 +5,7 @@ href: Image-chat-with-agent.md - name: Create agent with tools - href: Create-agent-with-tools.md \ No newline at end of file + href: Create-agent-with-tools.md + +- name: Use AutoGen.Net agent as model in AG Studio + href: Use-AutoGen.Net-agent-as-model-in-AG-Studio.md \ No newline at end of file From c19f44f626942262be5c0f5ec30487d07e8db226 Mon Sep 17 00:00:00 2001 From: Yiran Wu <32823396+yiranwu0@users.noreply.github.com> Date: Sun, 21 Jul 2024 14:07:13 -0700 Subject: [PATCH 55/57] update (#3175) Co-authored-by: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> --- autogen/oai/client.py | 2 +- test/oai/test_client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/autogen/oai/client.py b/autogen/oai/client.py index ef3a3fd2b1b3..4e9d794a1f75 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -795,7 +795,7 @@ def _cost_with_customized_price( n_output_tokens = response.usage.completion_tokens if response.usage is not None else 0 # type: ignore [union-attr] if n_output_tokens is None: n_output_tokens = 0 - return n_input_tokens * price_1k[0] + n_output_tokens * price_1k[1] + return (n_input_tokens * price_1k[0] + n_output_tokens * price_1k[1]) / 1000 @staticmethod def _update_dict_from_chunk(chunk: BaseModel, d: Dict[str, Any], field: str) -> int: diff --git a/test/oai/test_client.py b/test/oai/test_client.py index debad5fae3b0..ea6f7ba7c5f7 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -140,10 +140,10 @@ def test_customized_cost(): env_or_file=OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"tags": ["gpt-3.5-turbo-instruct"]} ) for config in config_list: - config.update({"price": [1, 1]}) + config.update({"price": [1000, 1000]}) client = OpenAIWrapper(config_list=config_list, cache_seed=None) response = client.create(prompt="1+3=") - assert response.cost >= 4, "Due to customized pricing, cost should be greater than 4" + assert response.cost >= 4 and response.cost < 10, "Due to customized pricing, cost should be > 4 and < 10" @pytest.mark.skipif(skip, reason="openai>=1 not installed") From 06e6b3ff05636526c0b26c094e3cc491e7ee3e24 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 22 Jul 2024 09:43:58 -0700 Subject: [PATCH 56/57] [.Net] Allow passing a kernel to Interactive Service. (#3183) * accept a running kernel for Interactive Service * add kernel running check --- .../InteractiveService.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs b/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs index 7490b64e1267..1ca19fcbcfff 100644 --- a/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs +++ b/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs @@ -19,7 +19,7 @@ public class InteractiveService : IDisposable private bool disposedValue; private const string DotnetInteractiveToolNotInstallMessage = "Cannot find a tool in the manifest file that has a command named 'dotnet-interactive'."; //private readonly ProcessJobTracker jobTracker = new ProcessJobTracker(); - private string installingDirectory; + private string? installingDirectory; public event EventHandler? DisplayEvent; @@ -30,7 +30,11 @@ public class InteractiveService : IDisposable public event EventHandler? HoverTextProduced; /// - /// Create an instance of InteractiveService + /// Install dotnet interactive tool to + /// and create an instance of . + /// + /// When using this constructor, you need to call to install dotnet interactive tool + /// and start the kernel. /// /// dotnet interactive installing directory public InteractiveService(string installingDirectory) @@ -38,8 +42,23 @@ public InteractiveService(string installingDirectory) this.installingDirectory = installingDirectory; } + /// + /// Create an instance of with a running kernel. + /// When using this constructor, you don't need to call to start the kernel. + /// + /// + public InteractiveService(Kernel kernel) + { + this.kernel = kernel; + } + public async Task StartAsync(string workingDirectory, CancellationToken ct = default) { + if (this.kernel != null) + { + return true; + } + this.kernel = await this.CreateKernelAsync(workingDirectory, true, ct); return true; } From 1daf852f8639b69fd9296886de8aec1a33ba3d80 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Mon, 22 Jul 2024 09:44:12 -0700 Subject: [PATCH 57/57] rename Service -> WebAPI (#3177) --- dotnet/sample/AutoGen.WebAPI.Sample/Program.cs | 2 +- dotnet/src/AutoGen.WebAPI/Extension.cs | 2 +- .../AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs | 2 +- .../src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs | 2 +- .../AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs | 2 +- .../AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs | 2 +- .../AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs | 2 +- .../AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs | 2 +- .../src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs | 2 +- .../AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs | 2 +- .../OpenAI/Service/OpenAIChatCompletionService.cs | 2 +- dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs | 4 ++-- dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs | 2 +- .../OpenAIChatCompletionMiddlewareTests.cs | 2 +- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs b/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs index 71b8b89219aa..dbeb8494363d 100644 --- a/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs +++ b/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs @@ -3,7 +3,7 @@ using System.Runtime.CompilerServices; using AutoGen.Core; -using AutoGen.Service; +using AutoGen.WebAPI; var alice = new DummyAgent("alice"); var bob = new DummyAgent("bob"); diff --git a/dotnet/src/AutoGen.WebAPI/Extension.cs b/dotnet/src/AutoGen.WebAPI/Extension.cs index 30fd8843174d..c8534e43e540 100644 --- a/dotnet/src/AutoGen.WebAPI/Extension.cs +++ b/dotnet/src/AutoGen.WebAPI/Extension.cs @@ -4,7 +4,7 @@ using AutoGen.Core; using Microsoft.AspNetCore.Builder; -namespace AutoGen.Service; +namespace AutoGen.WebAPI; public static class Extension { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs index 78e7b8a6dd0a..888a0f8dd8c8 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs @@ -5,7 +5,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIMessageConverter : JsonConverter { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs index 62771d58a28c..bfd090358453 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIAssistantMessage : OpenAIMessage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs index c5fa3334c5b8..041f4cfc848c 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIChatCompletion { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs index 82ba56a6adf2..35b6fce59a8e 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIChatCompletionChoice { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs index 8699523f792b..de6be0dbf7a5 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIChatCompletionMessage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs index 6e656ded8459..0b9137d43a39 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIChatCompletionOption { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs index 299d62c9d9e0..f196ccb842ea 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIChatCompletionUsage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs index 081620c414df..a50012c9fed1 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIImageUrlObject { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs index 24cbb72fb6c0..deb729b72003 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; [JsonConverter(typeof(OpenAIMessageConverter))] internal abstract class OpenAIMessage diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs index 7a49851446be..e95991388b7f 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIStreamOptions { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs index 57a671b8cdfa..f29b10826c4f 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAISystemMessage : OpenAIMessage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs index 7d003f2d797e..f3fc37f9c44f 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIToolCallObject { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs index 5a0bb968d8b5..0c84c164cd96 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIToolMessage : OpenAIMessage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs index c6ada9d8f73f..28b83ffb3058 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIUserImageContent : OpenAIUserMessageItem { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs index 17cb529daf60..b5f1e7c50c12 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIUserMessage : OpenAIMessage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs index af363c1c90ba..94e7d91534a5 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal abstract class OpenAIUserMessageItem { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs index acb8279e7938..789df5afaaae 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIUserMultiModalMessage : OpenAIMessage { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs index fdb122253089..d22d5aa4c7f3 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace AutoGen.Service.OpenAI.DTO; +namespace AutoGen.WebAPI.OpenAI.DTO; internal class OpenAIUserTextContent : OpenAIUserMessageItem { diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs index b84f62c7ce20..27481da006a2 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Threading.Tasks; using AutoGen.Core; -using AutoGen.Service.OpenAI.DTO; +using AutoGen.WebAPI.OpenAI.DTO; namespace AutoGen.Server; diff --git a/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs b/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs index 3563a6dc2bc3..53b3699fd62e 100644 --- a/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs +++ b/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs @@ -5,10 +5,10 @@ using System.Threading.Tasks; using AutoGen.Core; using AutoGen.Server; -using AutoGen.Service.OpenAI.DTO; +using AutoGen.WebAPI.OpenAI.DTO; using Microsoft.AspNetCore.Http; -namespace AutoGen.Service; +namespace AutoGen.WebAPI; public class OpenAIChatCompletionMiddleware : Microsoft.AspNetCore.Http.IMiddleware { diff --git a/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs index 6f9d217b50c2..957f8d1d799b 100644 --- a/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs +++ b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs @@ -4,7 +4,7 @@ using System.Runtime.CompilerServices; using AutoGen.Core; -namespace AutoGen.Service.Tests; +namespace AutoGen.WebAPI.Tests; public class EchoAgent : IStreamingAgent { diff --git a/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs b/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs index bb0f6048e9a8..07bdc850936d 100644 --- a/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs +++ b/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs @@ -12,7 +12,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -namespace AutoGen.Service.Tests; +namespace AutoGen.WebAPI.Tests; public class OpenAIChatCompletionMiddlewareTests {