LLM micro service extract input model name

No input model from UI. vLLM and OLLAMA get the model name from the envrionment TGI mode model form ModelID Signed-off-by: Wang, Xigui <[email protected]>
opea-project · Aug 12, 2024 · 7dfa9d5 · 7dfa9d5
1 parent 5262d05
commit 7dfa9d5
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/comps/llms/text-generation/ollama/llm.py b/comps/llms/text-generation/ollama/llm.py
@@ -21,7 +21,7 @@
 def llm_generate(input: LLMParamsDoc):
     ollama = Ollama(
         base_url=ollama_endpoint,
-        model=input.model,
+        model=input.model if input.model else model_name,
         num_predict=input.max_new_tokens,
         top_k=input.top_k,
         top_p=input.top_p,
@@ -49,4 +49,5 @@ async def stream_generator():
 
 if __name__ == "__main__":
     ollama_endpoint = os.getenv("OLLAMA_ENDPOINT", "http://localhost:11434")
+    model_name = os.getenv("OLLAMA_MODEL", "meta-llama/Meta-Llama-3-8B-Instruct")
     opea_microservices["opea_service@llm_ollama"].start()