From 7d07db99a66ab785026d809f499bd8c779b589cd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 Jan 2025 17:24:18 -0800 Subject: [PATCH] (proxy - RPS) - Get 2K RPS at 4 instances, minor fix `aiohttp_openai/` (#7659) * speed up transform_response * use 2 workers * undo changes to uvicorn * ci/cd run again --- litellm/llms/aiohttp_openai/chat/transformation.py | 12 ++++++++++-- tests/llm_translation/test_aiohttp_openai.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index b157d230efc2..53157ad113af 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -13,7 +13,7 @@ from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import ModelResponse +from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -66,4 +66,12 @@ async def transform_response( # type: ignore json_mode: Optional[bool] = None, ) -> ModelResponse: _json_response = await raw_response.json() - return ModelResponse(**_json_response) + model_response.id = _json_response.get("id") + model_response.choices = [ + Choices(**choice) for choice in _json_response.get("choices") + ] + model_response.created = _json_response.get("created") + model_response.model = _json_response.get("model") + model_response.object = _json_response.get("object") + model_response.system_fingerprint = _json_response.get("system_fingerprint") + return model_response diff --git a/tests/llm_translation/test_aiohttp_openai.py b/tests/llm_translation/test_aiohttp_openai.py index 5d29d117da8a..5b92c924ec74 100644 --- a/tests/llm_translation/test_aiohttp_openai.py +++ b/tests/llm_translation/test_aiohttp_openai.py @@ -6,7 +6,7 @@ sys.path.insert( 0, os.path.abspath("../../") -) # Adds the parent directory to the system path +) # Adds the parent directory to the system-path import litellm