set default maxTokens for autocomplete in CompletionProvider and llmF…

…romDescription
continuedev · Mar 3, 2025 · 7eeb5c6 · 7eeb5c6
1 parent ceede79
commit 7eeb5c6
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 8 deletions.
diff --git a/core/autocomplete/CompletionProvider.ts b/core/autocomplete/CompletionProvider.ts
@@ -175,12 +175,20 @@ export class CompletionProvider {
         this.ide.getWorkspaceDirs(),
       ]);
 
-      const { prompt, prefix, suffix, completionOptions } = renderPrompt({
+      const { prompt, prefix, suffix, completionOptions: _completionOptions } = renderPrompt({
         snippetPayload,
         workspaceDirs,
         helper,
       });
 
+      // Default maxTokens for autocomplete set in core/llm/llms/index.ts llmFromDescription()
+      const completionOptions = {
+        ..._completionOptions,
+        maxTokens: _completionOptions?.maxTokens ||
+          llm.completionOptions.autoCompleteMaxTokens ||
+          llm.completionOptions.maxTokens
+      };
+
       // Completion
       let completion: string | undefined = "";
 
@@ -220,11 +228,11 @@ export class CompletionProvider {
 
         const processedCompletion = helper.options.transform
           ? postprocessCompletion({
-              completion,
-              prefix: helper.prunedPrefix,
-              suffix: helper.prunedSuffix,
-              llm,
-            })
+            completion,
+            prefix: helper.prunedPrefix,
+            suffix: helper.prunedSuffix,
+            llm,
+          })
           : completion;
 
         completion = processedCompletion;

diff --git a/core/index.d.ts b/core/index.d.ts
@@ -909,6 +909,7 @@ export interface BaseCompletionOptions {
   mirostat?: number;
   stop?: string[];
   maxTokens?: number;
+  autoCompleteMaxTokens?: number;
   numThreads?: number;
   useMmap?: boolean;
   keepAlive?: number;

diff --git a/core/llm/llms/index.ts b/core/llm/llms/index.ts
@@ -19,7 +19,6 @@ import Cohere from "./Cohere";
 import DeepInfra from "./DeepInfra";
 import Deepseek from "./Deepseek";
 import Fireworks from "./Fireworks";
-import NCompass from "./NCompass";
 import Flowise from "./Flowise";
 import FreeTrial from "./FreeTrial";
 import FunctionNetwork from "./FunctionNetwork";
@@ -35,7 +34,9 @@ import Mistral from "./Mistral";
 import MockLLM from "./Mock";
 import Moonshot from "./Moonshot";
 import Msty from "./Msty";
+import NCompass from "./NCompass";
 import Nebius from "./Nebius";
+import Novita from "./Novita";
 import Nvidia from "./Nvidia";
 import Ollama from "./Ollama";
 import OpenAI from "./OpenAI";
@@ -49,7 +50,6 @@ import ContinueProxy from "./stubs/ContinueProxy";
 import TestLLM from "./Test";
 import TextGenWebUI from "./TextGenWebUI";
 import Together from "./Together";
-import Novita from "./Novita";
 import VertexAI from "./VertexAI";
 import Vllm from "./Vllm";
 import WatsonX from "./WatsonX";
@@ -112,6 +112,7 @@ export async function llmFromDescription(
   writeLog: (log: string) => Promise<void>,
   completionOptions?: BaseCompletionOptions,
   systemMessage?: string,
+  isAutocomplete = false
 ): Promise<BaseLLM | undefined> {
   const cls = LLMClasses.find((llm) => llm.providerName === desc.provider);
 
@@ -137,6 +138,11 @@ export async function llmFromDescription(
       maxTokens:
         finalCompletionOptions.maxTokens ??
         cls.defaultOptions?.completionOptions?.maxTokens,
+      autoCompleteMaxTokens:
+        finalCompletionOptions.autoCompleteMaxTokens ??
+        finalCompletionOptions.maxTokens ??
+        cls.defaultOptions?.completionOptions?.autoCompleteMaxTokens ??
+        256
     },
     systemMessage,
     writeLog,