Add support for searXNG search for agents (Mintplex-Labs#1733)

resolves Mintplex-Labs#1367

.vscode/settings.json

@@ -32,7 +32,9 @@
     "opendocument",
     "openrouter",
     "Qdrant",
+    "searxng",
     "Serper",
+    "Serply",
     "textgenwebui",
     "togetherai",
     "vectordbs",

docker/.env.example

@@ -245,3 +245,6 @@ GID='1000'
 
 #------ Serply.io ----------- https://serply.io/
 # AGENT_SERPLY_API_KEY=
+
+#------ SearXNG ----------- https://github.com/searxng/searxng
+# AGENT_SEARXNG_API_URL=

frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx

@@ -182,3 +182,25 @@ export function SerplySearchOptions({ settings }) {
     </>
   );
 }
+
+export function SearXNGOptions({ settings }) {
+  return (
+    <div className="flex gap-x-4">
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          SearXNG API base URL
+        </label>
+        <input
+          type="url"
+          name="env::AgentSearXNGApiUrl"
+          className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="SearXNG API Key"
+          defaultValue={settings?.AgentSearXNGApiUrl}
+          required={true}
+          autoComplete="off"
+          spellCheck={false}
+        />
+      </div>
+    </div>
+  );
+}

frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx

@@ -4,6 +4,7 @@ import GoogleSearchIcon from "./icons/google.png";
 import SerperDotDevIcon from "./icons/serper.png";
 import BingSearchIcon from "./icons/bing.png";
 import SerplySearchIcon from "./icons/serply.png";
+import SearXNGSearchIcon from "./icons/searxng.png";
 import {
   CaretUpDown,
   MagnifyingGlass,
@@ -17,6 +18,7 @@ import {
   GoogleSearchOptions,
   BingSearchOptions,
   SerplySearchOptions,
+  SearXNGOptions,
 } from "./SearchProviderOptions";
 
 const SEARCH_PROVIDERS = [
@@ -60,6 +62,14 @@ const SEARCH_PROVIDERS = [
     description:
       "Serply.io web-search. Free account with a 100 calls/month forever.",
   },
+  {
+    name: "SearXNG",
+    value: "searxng-engine",
+    logo: SearXNGSearchIcon,
+    options: (settings) => <SearXNGOptions settings={settings} />,
+    description:
+      "Free, open-source, internet meta-search engine with no tracking.",
+  },
 ];
 
 export default function AgentWebSearchSelection({

server/.env.example

@@ -241,3 +241,6 @@ TTS_PROVIDER="native"
 
 #------ Serply.io ----------- https://serply.io/
 # AGENT_SERPLY_API_KEY=
+
+#------ SearXNG ----------- https://github.com/searxng/searxng
+# AGENT_SEARXNG_API_URL=

server/models/systemSettings.js

@@ -76,6 +76,7 @@ const SystemSettings = {
             "serper-dot-dev",
             "bing-search",
             "serply-engine",
+            "searxng-engine",
           ].includes(update)
         )
           throw new Error("Invalid SERP provider.");
@@ -176,10 +177,11 @@ const SystemSettings = {
       // Agent Settings & Configs
       // --------------------------------------------------------
       AgentGoogleSearchEngineId: process.env.AGENT_GSE_CTX || null,
-      AgentGoogleSearchEngineKey: process.env.AGENT_GSE_KEY || null,
-      AgentSerperApiKey: process.env.AGENT_SERPER_DEV_KEY || null,
-      AgentBingSearchApiKey: process.env.AGENT_BING_SEARCH_API_KEY || null,
-      AgentSerplyApiKey: process.env.AGENT_SERPLY_API_KEY || null,
+      AgentGoogleSearchEngineKey: !!process.env.AGENT_GSE_KEY || null,
+      AgentSerperApiKey: !!process.env.AGENT_SERPER_DEV_KEY || null,
+      AgentBingSearchApiKey: !!process.env.AGENT_BING_SEARCH_API_KEY || null,
+      AgentSerplyApiKey: !!process.env.AGENT_SERPLY_API_KEY || null,
+      AgentSearXNGApiUrl: process.env.AGENT_SEARXNG_API_URL || null,
     };
   },
 

server/utils/agents/aibitat/plugins/web-browsing.js

@@ -71,6 +71,9 @@ const webBrowsing = {
               case "serply-engine":
                 engine = "_serplyEngine";
                 break;
+              case "searxng-engine":
+                engine = "_searXNGEngine";
+                break;
               default:
                 engine = "_googleSearchEngine";
             }
@@ -102,7 +105,7 @@ const webBrowsing = {
                 query.length > 100 ? `${query.slice(0, 100)}...` : query
               }"`
             );
-            const searchResponse = await fetch(searchURL)
+            const data = await fetch(searchURL)
               .then((res) => res.json())
               .then((searchResult) => searchResult?.items || [])
               .then((items) => {
@@ -116,10 +119,15 @@ const webBrowsing = {
               })
               .catch((e) => {
                 console.log(e);
-                return {};
+                return [];
               });
 
-            return JSON.stringify(searchResponse);
+            if (data.length === 0)
+              return `No information was found online for the search query.`;
+            this.super.introspect(
+              `${this.caller}: I found ${data.length} results - looking over them now.`
+            );
+            return JSON.stringify(data);
           },
 
           /**
@@ -176,6 +184,9 @@ const webBrowsing = {
 
             if (data.length === 0)
               return `No information was found online for the search query.`;
+            this.super.introspect(
+              `${this.caller}: I found ${data.length} results - looking over them now.`
+            );
             return JSON.stringify(data);
           },
           _bingWebSearch: async function (query) {
@@ -219,6 +230,9 @@ const webBrowsing = {
 
             if (searchResponse.length === 0)
               return `No information was found online for the search query.`;
+            this.super.introspect(
+              `${this.caller}: I found ${data.length} results - looking over them now.`
+            );
             return JSON.stringify(searchResponse);
           },
           _serplyEngine: async function (
@@ -293,6 +307,71 @@ const webBrowsing = {
 
             if (data.length === 0)
               return `No information was found online for the search query.`;
+            this.super.introspect(
+              `${this.caller}: I found ${data.length} results - looking over them now.`
+            );
+            return JSON.stringify(data);
+          },
+          _searXNGEngine: async function (query) {
+            let searchURL;
+            if (!process.env.AGENT_SEARXNG_API_URL) {
+              this.super.introspect(
+                `${this.caller}: I can't use SearXNG searching because the user has not defined the required base URL.\nPlease set this value in the agent skill settings.`
+              );
+              return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
+            }
+
+            try {
+              searchURL = new URL(process.env.AGENT_SEARXNG_API_URL);
+              searchURL.searchParams.append("q", encodeURIComponent(query));
+              searchURL.searchParams.append("format", "json");
+            } catch (e) {
+              this.super.handlerProps.log(`SearXNG Search: ${e.message}`);
+              this.super.introspect(
+                `${this.caller}: I can't use SearXNG searching because the url provided is not a valid URL.`
+              );
+              return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
+            }
+
+            this.super.introspect(
+              `${this.caller}: Using SearXNG to search for "${
+                query.length > 100 ? `${query.slice(0, 100)}...` : query
+              }"`
+            );
+
+            const { response, error } = await fetch(searchURL.toString(), {
+              method: "GET",
+              headers: {
+                "Content-Type": "application/json",
+                "User-Agent": "anything-llm",
+              },
+            })
+              .then((res) => res.json())
+              .then((data) => {
+                return { response: data, error: null };
+              })
+              .catch((e) => {
+                return { response: null, error: e.message };
+              });
+            if (error)
+              return `There was an error searching for content. ${error}`;
+
+            const data = [];
+            response.results?.forEach((searchResult) => {
+              const { url, title, content, publishedDate } = searchResult;
+              data.push({
+                title,
+                link: url,
+                snippet: content,
+                publishedDate,
+              });
+            });
+
+            if (data.length === 0)
+              return `No information was found online for the search query.`;
+            this.super.introspect(
+              `${this.caller}: I found ${data.length} results - looking over them now.`
+            );
             return JSON.stringify(data);
           },
         });

server/utils/helpers/updateENV.js

@@ -407,6 +407,10 @@ const KEY_MAPPING = {
     envKey: "AGENT_SERPLY_API_KEY",
     checks: [],
   },
+  AgentSearXNGApiUrl: {
+    envKey: "AGENT_SEARXNG_API_URL",
+    checks: [],
+  },
 
   // TTS/STT Integration ENVS
   TextToSpeechProvider: {