From fe3b5bbc23a99533bc7d4a94ae073828ed025974 Mon Sep 17 00:00:00 2001 From: Elinx Date: Wed, 17 Apr 2024 19:07:23 +0800 Subject: [PATCH] [Bugfix] fix output parsing error for trtllm backend (#4137) Co-authored-by: Roger Wang --- benchmarks/backend_request_func.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index bab570252c929..f9d167590fe47 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -135,6 +135,7 @@ async def async_request_trt_llm( "data:") data = json.loads(chunk) + output.generated_text += data["text_output"] timestamp = time.perf_counter() # First token if ttft == 0.0: @@ -149,7 +150,6 @@ async def async_request_trt_llm( most_recent_timestamp = timestamp output.latency = most_recent_timestamp - st - output.generated_text = json.loads(data)["text_output"] output.success = True else: