default maxTokens setting for autocomplete

core/autocomplete/CompletionProvider.ts

@@ -5,7 +5,7 @@
 import { DEFAULT_AUTOCOMPLETE_OPTS } from "../util/parameters.js";
 
 import { shouldCompleteMultiline } from "./classification/shouldCompleteMultiline.js";
 import { ContextRetrievalService } from "./context/ContextRetrievalService.js";
 
 import { BracketMatchingService } from "./filtering/BracketMatchingService.js";
 import { CompletionStreamer } from "./generation/CompletionStreamer.js";
@@ -175,12 +175,20 @@
         this.ide.getWorkspaceDirs(),
       ]);
 
-      const { prompt, prefix, suffix, completionOptions } = renderPrompt({
+      const { prompt, prefix, suffix, completionOptions: _completionOptions } = renderPrompt({
         snippetPayload,
         workspaceDirs,
         helper,
       });
 
+      // Default maxTokens for autocomplete set in core/llm/llms/index.ts llmFromDescription()
+      const completionOptions = {
+        ..._completionOptions,
+        maxTokens: _completionOptions?.maxTokens ||
+          llm.completionOptions.autoCompleteMaxTokens ||
+          llm.completionOptions.maxTokens
+      };
+
       // Completion
       let completion: string | undefined = "";
 
@@ -220,11 +228,11 @@
 
         const processedCompletion = helper.options.transform
           ? postprocessCompletion({
-              completion,
-              prefix: helper.prunedPrefix,
-              suffix: helper.prunedSuffix,
-              llm,
-            })
+            completion,
+            prefix: helper.prunedPrefix,
+            suffix: helper.prunedSuffix,
+            llm,
+          })
           : completion;
 
         completion = processedCompletion;
@@ -257,7 +265,7 @@
 
       // Save to cache
       if (!outcome.cacheHit && helper.options.useCache) {
         (await this.autocompleteCache).put(outcome.prefix, outcome.completion);
       }
 
       // When using the JetBrains extension, Mark as displayed

core/index.d.ts

@@ -909,6 +909,7 @@ export interface BaseCompletionOptions {
   mirostat?: number;
   stop?: string[];
   maxTokens?: number;
+  autoCompleteMaxTokens?: number;
   numThreads?: number;
   useMmap?: boolean;
   keepAlive?: number;

core/llm/llms/index.ts

@@ -19,7 +19,6 @@ import Cohere from "./Cohere";
 import DeepInfra from "./DeepInfra";
 import Deepseek from "./Deepseek";
 import Fireworks from "./Fireworks";
-import NCompass from "./NCompass";
 import Flowise from "./Flowise";
 import FreeTrial from "./FreeTrial";
 import FunctionNetwork from "./FunctionNetwork";
@@ -35,7 +34,9 @@ import Mistral from "./Mistral";
 import MockLLM from "./Mock";
 import Moonshot from "./Moonshot";
 import Msty from "./Msty";
+import NCompass from "./NCompass";
 import Nebius from "./Nebius";
+import Novita from "./Novita";
 import Nvidia from "./Nvidia";
 import Ollama from "./Ollama";
 import OpenAI from "./OpenAI";
@@ -49,7 +50,6 @@ import ContinueProxy from "./stubs/ContinueProxy";
 import TestLLM from "./Test";
 import TextGenWebUI from "./TextGenWebUI";
 import Together from "./Together";
-import Novita from "./Novita";
 import VertexAI from "./VertexAI";
 import Vllm from "./Vllm";
 import WatsonX from "./WatsonX";
@@ -112,6 +112,7 @@ export async function llmFromDescription(
   writeLog: (log: string) => Promise<void>,
   completionOptions?: BaseCompletionOptions,
   systemMessage?: string,
+  isAutocomplete = false
 ): Promise<BaseLLM | undefined> {
   const cls = LLMClasses.find((llm) => llm.providerName === desc.provider);
 
@@ -137,6 +138,11 @@ export async function llmFromDescription(
       maxTokens:
         finalCompletionOptions.maxTokens ??
         cls.defaultOptions?.completionOptions?.maxTokens,
+      autoCompleteMaxTokens:
+        finalCompletionOptions.autoCompleteMaxTokens ??
+        finalCompletionOptions.maxTokens ??
+        cls.defaultOptions?.completionOptions?.autoCompleteMaxTokens ??
+        256
     },
     systemMessage,
     writeLog,