From 839f258bbb053f31f41e98a699f7e348cc6c43b8 Mon Sep 17 00:00:00 2001
From: Tianjing Li <tianjinglimail@gmail.com>
Date: Fri, 13 Dec 2024 15:00:16 -0500
Subject: [PATCH] backend: Handle error when web scraping fails in Hybrid
 search (#879)

Handle error
---
 src/backend/tools/hybrid_search.py | 3 ++-
 src/backend/tools/web_scrape.py    | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/backend/tools/hybrid_search.py b/src/backend/tools/hybrid_search.py
index 849691c3cb..e9bceefba7 100644
--- a/src/backend/tools/hybrid_search.py
+++ b/src/backend/tools/hybrid_search.py
@@ -141,7 +141,8 @@ async def rerank_results(
                 documents=[
                     f"{result.get('title', '')} {result.get('text')}"
                     for result in results_batch
-                    if "text" in result
+                    if isinstance(result, dict)
+                    and "text" in result
                 ],
                 ctx=ctx,
             )
diff --git a/src/backend/tools/web_scrape.py b/src/backend/tools/web_scrape.py
index 849ac606ed..c82ae74c9a 100644
--- a/src/backend/tools/web_scrape.py
+++ b/src/backend/tools/web_scrape.py
@@ -64,15 +64,15 @@ async def call(
                     return await self.handle_response(response, url)
 
             except aiohttp.ClientError as e:
-                return  {
+                return  [{
                     "text": f"Client error using web scrape: {str(e)}",
                     "url": url,
-                }
+                }]
             except Exception as e:
-                return {
+                return [{
                     "text": f"Request failed using web scrape: {str(e)}",
                     "url": url,
-                }
+                }]
 
     async def handle_response(self, response: aiohttp.ClientResponse, url: str):
         content_type = response.headers.get("content-type")