Skip to content

Commit

Permalink
[FEAT] Auto detection for Ollama and LMStudio (Mintplex-Labs#1756)
Browse files Browse the repository at this point in the history 
* add ollama automatic url detection in llm and embedder prefrences

* implement auto detection for lmstudio llm and embedder/improve performance of checking common urls

* fix modal not clearing

* fix lmstudio url check

* improve ux for ollama llm provider option

* improve ux for lm studio llm provider option

* improve ux for ollama embedder option

* improve ux for lmstudio embedder option

* ux improvement lmstudio embedder options

* refactor implementation to hook and use native timeout
Swap to promise.any for resolving of available endpoints

* implement useProviderEndpointAutoDiscovery hook for lmstudio and ollama provider options

---------

Co-authored-by: timothycarambat <[email protected]>
  • Loading branch information
@shatfield4 @timothycarambat @franzbischoff
2 people authored and franzbischoff committed Jul 3, 2024
1 parent 4eccb85 commit be6dec4
Show file tree
Hide file tree
Showing 7 changed files with 596 additions and 160 deletions.
154 changes: 117 additions & 37 deletions frontend/src/components/EmbeddingSelection/LMStudioOptions/index.jsx
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,112 @@
import React, { useEffect, useState } from "react";
import System from "@/models/system";
import PreLoader from "@/components/Preloader";
import { LMSTUDIO_COMMON_URLS } from "@/utils/constants";
import { CaretDown, CaretUp } from "@phosphor-icons/react";
import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";

export default function LMStudioEmbeddingOptions({ settings }) {
const [basePathValue, setBasePathValue] = useState(
settings?.EmbeddingBasePath
const {
autoDetecting: loading,
basePath,
basePathValue,
showAdvancedControls,
setShowAdvancedControls,
handleAutoDetectClick,
} = useProviderEndpointAutoDiscovery({
provider: "lmstudio",
initialBasePath: settings?.EmbeddingBasePath,
ENDPOINTS: LMSTUDIO_COMMON_URLS,
});

const [maxChunkLength, setMaxChunkLength] = useState(
settings?.EmbeddingModelMaxChunkLength || 8192
);
const [basePath, setBasePath] = useState(settings?.EmbeddingBasePath);

const handleMaxChunkLengthChange = (e) => {
setMaxChunkLength(Number(e.target.value));
};

return (
<div className="w-full flex flex-col gap-y-4">
<div className="w-full flex items-center gap-4">
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
LMStudio Base URL
</label>
<input
type="url"
name="EmbeddingBasePath"
className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="http://localhost:1234/v1"
defaultValue={settings?.EmbeddingBasePath}
onChange={(e) => setBasePathValue(e.target.value)}
onBlur={() => setBasePath(basePathValue)}
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
<LMStudioModelSelection settings={settings} basePath={basePath} />
<div className="w-full flex items-start gap-4">
<LMStudioModelSelection settings={settings} basePath={basePath.value} />
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Max embedding chunk length
<label className="text-white text-sm font-semibold block mb-2">
Max Embedding Chunk Length
</label>
<input
type="number"
name="EmbeddingModelMaxChunkLength"
className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="8192"
min={1}
value={maxChunkLength}
onChange={handleMaxChunkLengthChange}
onScroll={(e) => e.target.blur()}
defaultValue={settings?.EmbeddingModelMaxChunkLength}
required={false}
required={true}
autoComplete="off"
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Maximum length of text chunks for embedding.
</p>
</div>
</div>
<div className="flex justify-start mt-4">
<button
onClick={(e) => {
e.preventDefault();
setShowAdvancedControls(!showAdvancedControls);
}}
className="text-white hover:text-white/70 flex items-center text-sm"
>
{showAdvancedControls ? "Hide" : "Show"} Manual Endpoint Input
{showAdvancedControls ? (
<CaretUp size={14} className="ml-1" />
) : (
<CaretDown size={14} className="ml-1" />
)}
</button>
</div>

<div hidden={!showAdvancedControls}>
<div className="w-full flex items-start gap-4">
<div className="flex flex-col w-60">
<div className="flex justify-between items-center mb-2">
<label className="text-white text-sm font-semibold">
LM Studio Base URL
</label>
{loading ? (
<PreLoader size="6" />
) : (
<>
{!basePathValue.value && (
<button
onClick={handleAutoDetectClick}
className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
>
Auto-Detect
</button>
)}
</>
)}
</div>
<input
type="url"
name="EmbeddingBasePath"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="http://localhost:1234/v1"
value={basePathValue.value}
required={true}
autoComplete="off"
spellCheck={false}
onChange={basePath.onChange}
onBlur={basePath.onBlur}
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Enter the URL where LM Studio is running.
</p>
</div>
</div>
</div>
</div>
Expand All @@ -55,14 +119,23 @@ function LMStudioModelSelection({ settings, basePath = null }) {

useEffect(() => {
async function findCustomModels() {
if (!basePath || !basePath.includes("/v1")) {
if (!basePath) {
setCustomModels([]);
setLoading(false);
return;
}
setLoading(true);
const { models } = await System.customModels("lmstudio", null, basePath);
setCustomModels(models || []);
try {
const { models } = await System.customModels(
"lmstudio",
null,
basePath
);
setCustomModels(models || []);
} catch (error) {
console.error("Failed to fetch custom models:", error);
setCustomModels([]);
}
setLoading(false);
}
findCustomModels();
Expand All @@ -71,28 +144,32 @@ function LMStudioModelSelection({ settings, basePath = null }) {
if (loading || customModels.length == 0) {
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Chat Model Selection
<label className="text-white text-sm font-semibold block mb-2">
LM Studio Embedding Model
</label>
<select
name="EmbeddingModelPref"
disabled={true}
className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
<option disabled={true} selected={true}>
{basePath?.includes("/v1")
? "-- loading available models --"
: "-- waiting for URL --"}
{!!basePath
? "--loading available models--"
: "Enter LM Studio URL first"}
</option>
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Select the LM Studio model for embeddings. Models will load after
entering a valid LM Studio URL.
</p>
</div>
);
}

return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Chat Model Selection
<label className="text-white text-sm font-semibold block mb-2">
LM Studio Embedding Model
</label>
<select
name="EmbeddingModelPref"
Expand All @@ -115,6 +192,9 @@ function LMStudioModelSelection({ settings, basePath = null }) {
</optgroup>
)}
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Choose the LM Studio model you want to use for generating embeddings.
</p>
</div>
);
}
Loading

0 comments on commit be6dec4

Please sign in to comment.