[Bugfix]if the content is started with ":"(response of ping), client …

…should i… (vllm-project#5303) Signed-off-by: Wang, Yi A <[email protected]> Co-authored-by: Roger Wang <[email protected]>
Isotr0py · Jun 13, 2024 · 8840753 · 8840753
1 parent 916d219
commit 8840753
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py
@@ -68,9 +68,13 @@ async def async_request_tgi(
                         chunk_bytes = chunk_bytes.strip()
                         if not chunk_bytes:
                             continue
+                        chunk_bytes = chunk_bytes.decode("utf-8")
 
-                        chunk = remove_prefix(chunk_bytes.decode("utf-8"),
-                                              "data:")
+                        #NOTE: Sometimes TGI returns a ping response without
+                        # any data, we should skip it.
+                        if chunk_bytes.startswith(":"):
+                            continue
+                        chunk = remove_prefix(chunk_bytes, "data:")
 
                         data = json.loads(chunk)
                         timestamp = time.perf_counter()