forked from Mintplex-Labs/anything-llm
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[FEAT] Auto detection for Ollama and LMStudio (Mintplex-Labs#1756)
* add ollama automatic url detection in llm and embedder prefrences * implement auto detection for lmstudio llm and embedder/improve performance of checking common urls * fix modal not clearing * fix lmstudio url check * improve ux for ollama llm provider option * improve ux for lm studio llm provider option * improve ux for ollama embedder option * improve ux for lmstudio embedder option * ux improvement lmstudio embedder options * refactor implementation to hook and use native timeout Swap to promise.any for resolving of available endpoints * implement useProviderEndpointAutoDiscovery hook for lmstudio and ollama provider options --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2 people
authored and
sync_forks
committed
Jul 31, 2024
1 parent
4c6775d
commit ce096e6
Showing
7 changed files
with
596 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| import { useEffect, useState } from "react"; | ||
| import System from "@/models/system"; | ||
| import showToast from "@/utils/toast"; | ||
|
|
||
| export default function useProviderEndpointAutoDiscovery({ | ||
| provider = null, | ||
| initialBasePath = "", | ||
| ENDPOINTS = [], | ||
| }) { | ||
| const [loading, setLoading] = useState(false); | ||
| const [basePath, setBasePath] = useState(initialBasePath); | ||
| const [basePathValue, setBasePathValue] = useState(initialBasePath); | ||
| const [autoDetectAttempted, setAutoDetectAttempted] = useState(false); | ||
| const [showAdvancedControls, setShowAdvancedControls] = useState(true); | ||
|
|
||
| async function autoDetect(isInitialAttempt = false) { | ||
| setLoading(true); | ||
| setAutoDetectAttempted(true); | ||
| const possibleEndpoints = []; | ||
| ENDPOINTS.forEach((endpoint) => { | ||
| possibleEndpoints.push( | ||
| new Promise((resolve, reject) => { | ||
| System.customModels(provider, null, endpoint, 2_000) | ||
| .then((results) => { | ||
| if (!results?.models || results.models.length === 0) | ||
| throw new Error("No models"); | ||
| resolve({ endpoint, models: results.models }); | ||
| }) | ||
| .catch(() => { | ||
| reject(`${provider} @ ${endpoint} did not resolve.`); | ||
| }); | ||
| }) | ||
| ); | ||
| }); | ||
|
|
||
| const { endpoint, models } = await Promise.any(possibleEndpoints) | ||
| .then((resolved) => resolved) | ||
| .catch(() => { | ||
| console.error("All endpoints failed to resolve."); | ||
| return { endpoint: null, models: null }; | ||
| }); | ||
|
|
||
| if (models !== null) { | ||
| setBasePath(endpoint); | ||
| setBasePathValue(endpoint); | ||
| setLoading(false); | ||
| showToast("Provider endpoint discovered automatically.", "success", { | ||
| clear: true, | ||
| }); | ||
| setShowAdvancedControls(false); | ||
| return; | ||
| } | ||
|
|
||
| setLoading(false); | ||
| setShowAdvancedControls(true); | ||
| showToast( | ||
| "Couldn't automatically discover the provider endpoint. Please enter it manually.", | ||
| "info", | ||
| { clear: true } | ||
| ); | ||
| } | ||
|
|
||
| function handleAutoDetectClick(e) { | ||
| e.preventDefault(); | ||
| autoDetect(); | ||
| } | ||
|
|
||
| function handleBasePathChange(e) { | ||
| const value = e.target.value; | ||
| setBasePathValue(value); | ||
| } | ||
|
|
||
| function handleBasePathBlur() { | ||
| setBasePath(basePathValue); | ||
| } | ||
|
|
||
| useEffect(() => { | ||
| if (!initialBasePath && !autoDetectAttempted) autoDetect(true); | ||
| }, [initialBasePath, autoDetectAttempted]); | ||
|
|
||
| return { | ||
| autoDetecting: loading, | ||
| autoDetectAttempted, | ||
| showAdvancedControls, | ||
| setShowAdvancedControls, | ||
| basePath: { | ||
| value: basePath, | ||
| set: setBasePathValue, | ||
| onChange: handleBasePathChange, | ||
| onBlur: handleBasePathBlur, | ||
| }, | ||
| basePathValue: { | ||
| value: basePathValue, | ||
| set: setBasePathValue, | ||
| }, | ||
| handleAutoDetectClick, | ||
| runAutoDetect: autoDetect, | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters