From 3bc93cc9daeff5b0ecf2cc92ce66263dde96452a Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Wed, 28 Feb 2024 14:47:51 -0700 Subject: [PATCH 01/14] refactor/made it more typescript --- supabase/functions/chat/index.ts | 154 ++++++++++++++++++++----------- 1 file changed, 102 insertions(+), 52 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index 549412f..697f6d0 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -1,31 +1,60 @@ import { serve } from "https://deno.land/std@0.170.0/http/server.ts"; import OpenAI from "https://deno.land/x/openai@v4.26.0/mod.ts"; - import { corsHeaders } from "../common/cors.ts"; import { supabaseClient } from "../common/supabaseClient.ts"; import { ApplicationError, UserError } from "../common/errors.ts"; + +interface ChatClient { + chat: { + completions: { + create: (params: { model: string; messages: Message[] }) => Promise<{ choices: Choice[] }>; + }; + }; + embeddings: { + create: (params: { model: string; input: string }) => Promise<{ data: any[] }>; // Adjust the return type according to your actual data structure + }; +} + +interface SearchResult { + id: number; + raw_text: string; + similarity: number; +} + +interface Message { + role: string; + content: string; +} + +interface Choice { + message: string; +} + +// Current models available +type ModelName = "nousresearch/nous-capybara-34b" | "mistral" | "gpt-4-0125-preview"; + async function generateResponse( - useOpenRouter, - useOllama, - openaiClient, - openRouterClient, - ollamaClient, - messages + useOpenRouter: boolean, + useOllama: boolean, + openaiClient: ChatClient, + openRouterClient: ChatClient | null, + ollamaClient: ChatClient | null, + messages: Message[] ) { - let client; - let modelName; + let client: ChatClient; + let modelName: ModelName; - if (useOpenRouter) { + if (useOpenRouter && openRouterClient !== null) { client = openRouterClient; modelName = "nousresearch/nous-capybara-34b"; - } else if (useOllama) { + } else if (useOllama && ollamaClient !== null) { client = ollamaClient; modelName = "mistral"; } else { client = openaiClient; - modelName = "gpt-4-1106-preview"; + modelName = "gpt-4-0125-preview"; } const { choices } = await client.chat.completions.create({ @@ -36,13 +65,69 @@ async function generateResponse( return choices[0].message; } -const chat = async (req) => { +async function prepareChatMessagesWithEmbeddingRecords( + openaiClient: ChatClient, + supabase: any, + messageHistory: Message[] +): Promise { + + // Embed the last messageHistory message using OpenAI's embeddings API + const embeddingsResponse = await openaiClient.embeddings.create({ + model: "text-embedding-3-small", + input: messageHistory[messageHistory.length - 1].content, + }); + const embeddings = embeddingsResponse.data[0].embedding; + + // Retrieve records from Supabase based on embeddings similarity + const response = await supabase.rpc( + "match_records_embeddings_similarity", + { + query_embedding: JSON.stringify(embeddings), + match_threshold: 0.1, + match_count: 10, + } + ); + + if (response.error) { + console.log("recordsError: ", response.error); + throw new ApplicationError("Error getting records from Supabase"); + } + + const relevantRecords: SearchResult[] = response.data; + + // Ensure relevantRecords is not undefined or null + if (!relevantRecords) { + // Later we can do something more robust here, maybe some kind of retry mechanism, or have the AI respond. + throw new ApplicationError("No relevant records found"); + } + + console.log("relevantRecords: ", response.data); + + let messages = [ + { + role: "system", + content: `You are a helpful assistant, helping the user navigate through life. He is asking you questions, and you answer them with the best of your ability. + You have access to some of their records, to help you answer their question in a more personalized way. + + Records: + ${relevantRecords.map((record) => record.raw_text).join("\n")} + `, + }, + ...messageHistory, + ]; + + return messages; +} + +const chat = async (req: Request) => { if (req.method === "OPTIONS") { return new Response("ok", { headers: corsHeaders }); } const supabaseAuthToken = req.headers.get("Authorization") ?? ""; + if (!supabaseAuthToken) throw new ApplicationError("Missing supabase auth token"); + const supabase = supabaseClient(req, supabaseAuthToken); const { data: { user }, @@ -52,6 +137,7 @@ const chat = async (req) => { throw new ApplicationError( "Unable to get auth user details in request data" ); + const requestBody = await req.json(); const { messageHistory } = requestBody; @@ -64,7 +150,7 @@ const chat = async (req) => { const openRouterApiKey = Deno.env.get("OPENROUTER_API_KEY"); const useOpenRouter = Boolean(openRouterApiKey); // Use OpenRouter if API key is available - let openRouterClient; + let openRouterClient: ChatClient | null = null; if (useOpenRouter) { openRouterClient = new OpenAI({ baseURL: "https://openrouter.ai/api/v1", @@ -75,7 +161,7 @@ const chat = async (req) => { const ollamaApiKey = Deno.env.get("OLLAMA_BASE_URL"); const useOllama = Boolean(ollamaApiKey); // Use Ollama if OLLAMA_BASE_URL is available - let ollamaClient; + let ollamaClient: ChatClient | null = null; if (useOllama) { ollamaClient = new OpenAI({ baseURL: Deno.env.get("OLLAMA_BASE_URL"), @@ -83,43 +169,7 @@ const chat = async (req) => { }); } - - // Embed the last messageHistory message using OpenAI's embeddings API - const embeddingsResponse = await openaiClient.embeddings.create({ - model: "text-embedding-3-small", - input: messageHistory[messageHistory.length - 1].content, - }); - const embeddings = embeddingsResponse.data[0].embedding; - - // Retrieve records from Supabase based on embeddings similarity - const { data: relevantRecords, error: recordsError } = await supabase.rpc( - "match_records_embeddings_similarity", - { - query_embedding: JSON.stringify(embeddings), - match_threshold: 0.1, - match_count: 10, - } - ); - - if (recordsError) { - console.log("recordsError: ", recordsError); - throw new ApplicationError("Error getting records from Supabase"); - } - console.log("relevantRecords: ", relevantRecords); - - let messages = [ - { - role: "system", - content: `You are a helpful assistant, helping the user navigate through life. The user is asking you questions, and you answer them with the best of your ability. -You have access to some of their records, to help you answer their question in a more personalized way. - - - Records: - ${relevantRecords.map((r) => r.raw_text).join("\n")} - `, - }, - ...messageHistory, - ]; + const messages = await prepareChatMessagesWithEmbeddingRecords(openaiClient, supabase, messageHistory) try { const responseMessage = await generateResponse( From 1a70ce713c1ccdbaf9e2f3cc6f5a4357085c0dc5 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Wed, 6 Mar 2024 11:40:17 -0700 Subject: [PATCH 02/14] moved construction of message outside of function --- supabase/functions/chat/index.ts | 33 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index 697f6d0..4580f7c 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -65,7 +65,7 @@ async function generateResponse( return choices[0].message; } -async function prepareChatMessagesWithEmbeddingRecords( +async function getRelevantRecords( openaiClient: ChatClient, supabase: any, messageHistory: Message[] @@ -103,20 +103,8 @@ async function prepareChatMessagesWithEmbeddingRecords( console.log("relevantRecords: ", response.data); - let messages = [ - { - role: "system", - content: `You are a helpful assistant, helping the user navigate through life. He is asking you questions, and you answer them with the best of your ability. - You have access to some of their records, to help you answer their question in a more personalized way. - - Records: - ${relevantRecords.map((record) => record.raw_text).join("\n")} - `, - }, - ...messageHistory, - ]; - - return messages; + return relevantRecords; + } const chat = async (req: Request) => { @@ -169,7 +157,20 @@ const chat = async (req: Request) => { }); } - const messages = await prepareChatMessagesWithEmbeddingRecords(openaiClient, supabase, messageHistory) + const relevantRecords = await getRelevantRecords(openaiClient, supabase, messageHistory) + + let messages = [ + { + role: "system", + content: `You are a helpful assistant, helping the user navigate through life. He is asking you questions, and you answer them with the best of your ability. + You have access to some of their records, to help you answer their question in a more personalized way. + + Records: + ${relevantRecords.map((record) => record.raw_text).join("\n")} + `, + }, + ...messageHistory, + ]; try { const responseMessage = await generateResponse( From 527bdd0da53d3f503460ed5bdc15420bd891db75 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Wed, 28 Feb 2024 16:26:24 -0700 Subject: [PATCH 03/14] updated user message --- supabase/functions/chat/index.ts | 57 ++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index 4580f7c..2158677 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -34,6 +34,17 @@ interface Choice { // Current models available type ModelName = "nousresearch/nous-capybara-34b" | "mistral" | "gpt-4-0125-preview"; + +const openaiClient = new OpenAI({ + apiKey: Deno.env.get("OPENAI_API_KEY"), +}); +const openRouterApiKey = Deno.env.get("OPENROUTER_API_KEY"); +const useOpenRouter = Boolean(openRouterApiKey); // Use OpenRouter if API key is available +const ollamaApiKey = Deno.env.get("OLLAMA_BASE_URL"); +const useOllama = Boolean(ollamaApiKey); // Use Ollama if OLLAMA_BASE_URL is available +let openRouterClient: ChatClient | null = null; +let ollamaClient: ChatClient | null = null; + async function generateResponse( useOpenRouter: boolean, useOllama: boolean, @@ -68,16 +79,38 @@ async function generateResponse( async function getRelevantRecords( openaiClient: ChatClient, supabase: any, - messageHistory: Message[] + msgData: any, ): Promise { + const messageHistory = msgData.messageHistory; + const timestamp = msgData.timestamp; + + let lastMessage = [messageHistory[messageHistory.length - 1]]; + + const systemMessage = { + role: "system", + content: `Your objective is to determine the intent of the users message. Their requests can vary but will often be asking about their day, week, month and life. + Use the current date time ${timestamp} to help you answer their questions. + `, + }; + + const optimnizedUserMsg = await generateResponse( + useOpenRouter, + useOllama, + openaiClient, + openRouterClient, + ollamaClient, + messages + ); + console.log(timestamp); // Embed the last messageHistory message using OpenAI's embeddings API const embeddingsResponse = await openaiClient.embeddings.create({ model: "text-embedding-3-small", input: messageHistory[messageHistory.length - 1].content, }); const embeddings = embeddingsResponse.data[0].embedding; - + + console.log(messageHistory[messageHistory.length - 1]); // Retrieve records from Supabase based on embeddings similarity const response = await supabase.rpc( "match_records_embeddings_similarity", @@ -127,29 +160,17 @@ const chat = async (req: Request) => { ); const requestBody = await req.json(); - const { messageHistory } = requestBody; + const msgData = requestBody as { messageHistory: Message[]; timestamp: string }; - if (!messageHistory) throw new UserError("Missing query in request data"); - - const openaiClient = new OpenAI({ - apiKey: Deno.env.get("OPENAI_API_KEY"), - }); - - const openRouterApiKey = Deno.env.get("OPENROUTER_API_KEY"); - const useOpenRouter = Boolean(openRouterApiKey); // Use OpenRouter if API key is available - - let openRouterClient: ChatClient | null = null; + if (!msgData.messageHistory) throw new UserError("Missing query in request data"); + if (useOpenRouter) { openRouterClient = new OpenAI({ baseURL: "https://openrouter.ai/api/v1", apiKey: openRouterApiKey, }); } - - const ollamaApiKey = Deno.env.get("OLLAMA_BASE_URL"); - const useOllama = Boolean(ollamaApiKey); // Use Ollama if OLLAMA_BASE_URL is available - - let ollamaClient: ChatClient | null = null; + if (useOllama) { ollamaClient = new OpenAI({ baseURL: Deno.env.get("OLLAMA_BASE_URL"), From c392dc88bd863b9d1d37ff8ccb766ee4b1556180 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Thu, 29 Feb 2024 08:12:56 -0700 Subject: [PATCH 04/14] refactored openRouter logic --- supabase/functions/chat/index.ts | 61 ++++++++------------------------ 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index 2158677..f8b3c10 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -4,7 +4,6 @@ import { corsHeaders } from "../common/cors.ts"; import { supabaseClient } from "../common/supabaseClient.ts"; import { ApplicationError, UserError } from "../common/errors.ts"; - interface ChatClient { chat: { completions: { @@ -34,35 +33,33 @@ interface Choice { // Current models available type ModelName = "nousresearch/nous-capybara-34b" | "mistral" | "gpt-4-0125-preview"; - const openaiClient = new OpenAI({ apiKey: Deno.env.get("OPENAI_API_KEY"), }); -const openRouterApiKey = Deno.env.get("OPENROUTER_API_KEY"); -const useOpenRouter = Boolean(openRouterApiKey); // Use OpenRouter if API key is available -const ollamaApiKey = Deno.env.get("OLLAMA_BASE_URL"); -const useOllama = Boolean(ollamaApiKey); // Use Ollama if OLLAMA_BASE_URL is available -let openRouterClient: ChatClient | null = null; -let ollamaClient: ChatClient | null = null; +const useOpenRouter = Boolean(Deno.env.get("OPENROUTER_API_KEY")); // Use OpenRouter if API key is available +const useOllama = Boolean(Deno.env.get("OLLAMA_BASE_URL")); // Use Ollama if OLLAMA_BASE_URL is available async function generateResponse( useOpenRouter: boolean, useOllama: boolean, - openaiClient: ChatClient, - openRouterClient: ChatClient | null, - ollamaClient: ChatClient | null, messages: Message[] ) { let client: ChatClient; let modelName: ModelName; - if (useOpenRouter && openRouterClient !== null) { - client = openRouterClient; + if (useOpenRouter) { + client = new OpenAI({ + baseURL: "https://openrouter.ai/api/v1", + apiKey: Deno.env.get("OPENROUTER_API_KEY"), + }); modelName = "nousresearch/nous-capybara-34b"; - } else if (useOllama && ollamaClient !== null) { - client = ollamaClient; - modelName = "mistral"; + } else if (useOllama) { + client = new OpenAI({ + baseURL: Deno.env.get("OLLAMA_BASE_URL"), + apiKey: "ollama" + }); + modelName = "mistral"; } else { client = openaiClient; modelName = "gpt-4-0125-preview"; @@ -97,9 +94,6 @@ async function getRelevantRecords( const optimnizedUserMsg = await generateResponse( useOpenRouter, useOllama, - openaiClient, - openRouterClient, - ollamaClient, messages ); console.log(timestamp); @@ -141,9 +135,11 @@ async function getRelevantRecords( } const chat = async (req: Request) => { + if (req.method === "OPTIONS") { return new Response("ok", { headers: corsHeaders }); } + const supabaseAuthToken = req.headers.get("Authorization") ?? ""; if (!supabaseAuthToken) @@ -163,20 +159,6 @@ const chat = async (req: Request) => { const msgData = requestBody as { messageHistory: Message[]; timestamp: string }; if (!msgData.messageHistory) throw new UserError("Missing query in request data"); - - if (useOpenRouter) { - openRouterClient = new OpenAI({ - baseURL: "https://openrouter.ai/api/v1", - apiKey: openRouterApiKey, - }); - } - - if (useOllama) { - ollamaClient = new OpenAI({ - baseURL: Deno.env.get("OLLAMA_BASE_URL"), - apiKey: "ollama" - }); - } const relevantRecords = await getRelevantRecords(openaiClient, supabase, messageHistory) @@ -197,9 +179,6 @@ const chat = async (req: Request) => { const responseMessage = await generateResponse( useOpenRouter, useOllama, - openaiClient, - openRouterClient, - ollamaClient, messages ); @@ -216,16 +195,6 @@ const chat = async (req: Request) => { console.log("Error: ", error); throw new ApplicationError("Error processing chat completion"); } - - return new Response( - JSON.stringify({ - msg: { role: "assistant", content: "Hello from Deno Deploy!" }, - }), - { - headers: { ...corsHeaders, "Content-Type": "application/json" }, - status: 200, - } - ); }; serve(chat); From 0b419a52f9f85eb447d187072601d65875a04f7f Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Thu, 29 Feb 2024 10:32:51 -0700 Subject: [PATCH 05/14] backend streaming --- supabase/functions/chat/index.ts | 75 ++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index f8b3c10..d419ffe 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -7,11 +7,11 @@ import { ApplicationError, UserError } from "../common/errors.ts"; interface ChatClient { chat: { completions: { - create: (params: { model: string; messages: Message[] }) => Promise<{ choices: Choice[] }>; + create: (params: { model: string; messages: Message[]; stream?: boolean }) => AsyncIterable<{ choices: Choice[] }>; }; }; embeddings: { - create: (params: { model: string; input: string }) => Promise<{ data: any[] }>; // Adjust the return type according to your actual data structure + create: (params: { model: string; input: string }) => Promise<{ data: any[] }>; }; } @@ -27,7 +27,9 @@ interface Message { } interface Choice { - message: string; + delta: { + content: string; + }; } // Current models available @@ -39,12 +41,11 @@ const openaiClient = new OpenAI({ const useOpenRouter = Boolean(Deno.env.get("OPENROUTER_API_KEY")); // Use OpenRouter if API key is available const useOllama = Boolean(Deno.env.get("OLLAMA_BASE_URL")); // Use Ollama if OLLAMA_BASE_URL is available -async function generateResponse( +async function* generateResponse( useOpenRouter: boolean, useOllama: boolean, messages: Message[] ) { - let client: ChatClient; let modelName: ModelName; @@ -65,12 +66,15 @@ async function generateResponse( modelName = "gpt-4-0125-preview"; } - const { choices } = await client.chat.completions.create({ + const completion = await client.chat.completions.create({ model: modelName, messages, + stream: true, }); - console.log("Completion: ", choices[0]); - return choices[0].message; + + for await (const chunk of completion) { + yield chunk.choices[0].delta.content; + } } async function getRelevantRecords( @@ -94,17 +98,19 @@ async function getRelevantRecords( const optimnizedUserMsg = await generateResponse( useOpenRouter, useOllama, - messages - ); + lastMessage + ); console.log(timestamp); // Embed the last messageHistory message using OpenAI's embeddings API const embeddingsResponse = await openaiClient.embeddings.create({ model: "text-embedding-3-small", input: messageHistory[messageHistory.length - 1].content, }); + const embeddings = embeddingsResponse.data[0].embedding; console.log(messageHistory[messageHistory.length - 1]); + // Retrieve records from Supabase based on embeddings similarity const response = await supabase.rpc( "match_records_embeddings_similarity", @@ -146,14 +152,7 @@ const chat = async (req: Request) => { throw new ApplicationError("Missing supabase auth token"); const supabase = supabaseClient(req, supabaseAuthToken); - const { - data: { user }, - } = await supabase.auth.getUser(); - - if (!user) - throw new ApplicationError( - "Unable to get auth user details in request data" - ); + console.log("Supabase: ", supabase.auth); const requestBody = await req.json(); const msgData = requestBody as { messageHistory: Message[]; timestamp: string }; @@ -176,21 +175,33 @@ const chat = async (req: Request) => { ]; try { - const responseMessage = await generateResponse( - useOpenRouter, - useOllama, - messages - ); - - return new Response( - JSON.stringify({ - msg: responseMessage, - }), - { - headers: { ...corsHeaders, "Content-Type": "application/json" }, - status: 200, + const stream = new ReadableStream({ + async start(controller) { + try { + const responseMessageGenerator = generateResponse( + useOpenRouter, + useOllama, + messages + ); + + for await (const chunk of responseMessageGenerator) { + console.log("Chunk from AI:", chunk); + // Wrap the chunk in a JSON object + const jsonResponse = JSON.stringify({ message: chunk }) + "\n"; + const encodedChunk = new TextEncoder().encode(jsonResponse); + controller.enqueue(encodedChunk); + } + controller.close(); + } catch (error) { + console.error("Stream error:", error); + controller.error(error); + } } - ); + }); + + return new Response(stream, { + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); } catch (error) { console.log("Error: ", error); throw new ApplicationError("Error processing chat completion"); From bcf5d099c7dc1d47a246f71b8715fa83d89c87a7 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Thu, 29 Feb 2024 11:24:09 -0700 Subject: [PATCH 06/14] code clean up --- supabase/functions/chat/index.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index d419ffe..e80cb63 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -35,6 +35,7 @@ interface Choice { // Current models available type ModelName = "nousresearch/nous-capybara-34b" | "mistral" | "gpt-4-0125-preview"; + const openaiClient = new OpenAI({ apiKey: Deno.env.get("OPENAI_API_KEY"), }); @@ -152,7 +153,6 @@ const chat = async (req: Request) => { throw new ApplicationError("Missing supabase auth token"); const supabase = supabaseClient(req, supabaseAuthToken); - console.log("Supabase: ", supabase.auth); const requestBody = await req.json(); const msgData = requestBody as { messageHistory: Message[]; timestamp: string }; @@ -185,9 +185,7 @@ const chat = async (req: Request) => { ); for await (const chunk of responseMessageGenerator) { - console.log("Chunk from AI:", chunk); - // Wrap the chunk in a JSON object - const jsonResponse = JSON.stringify({ message: chunk }) + "\n"; + const jsonResponse = JSON.stringify({ token: chunk }) + "\n"; const encodedChunk = new TextEncoder().encode(jsonResponse); controller.enqueue(encodedChunk); } From 90af3f319a3ad675e292866d6b3aef2208b782c2 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Thu, 29 Feb 2024 11:25:41 -0700 Subject: [PATCH 07/14] backend streaming --- app/src/components/Chat.tsx | 73 +++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index 7abc19d..f9c9037 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -9,6 +9,7 @@ import NewConversationButton from './NewConversationButton'; import PromptForm from './PromptForm'; import SideMenu from './SideMenu'; import { ThemeToggle } from './ThemeToggle'; +import { useSupabaseConfig } from '../utils/useSupabaseConfig'; export default function Chat({ supabaseClient, @@ -23,6 +24,8 @@ export default function Chat({ const [conversationId, setConversationId] = useState(null); const [waitingForResponse, setWaitingForResponse] = useState(false); + const { supabaseUrl, supabaseToken } = useSupabaseConfig(); + const sendMessageAndReceiveResponse = useMutation({ mutationFn: async (userMessage: Message) => { const { data: sendMessageData, error: sendMessageError } = @@ -30,38 +33,54 @@ export default function Chat({ .from('conversations') .update({ context: [...messages, userMessage] }) .eq('id', conversationId); - + if (sendMessageError) throw sendMessageError; - + setMessages([...messages, userMessage]); setWaitingForResponse(true); - - const { data: aiResponseData, error: aiResponseError } = - await supabaseClient.functions.invoke('chat', { - body: { messageHistory: [...messages, userMessage] }, - }); - - if (aiResponseError) throw aiResponseError; - - const { data: updateConversationData, error: updateConversationError } = - await supabaseClient - .from('conversations') - .update({ context: [...messages, userMessage, aiResponseData.msg] }) - .eq('id', conversationId); - - if (updateConversationError) throw updateConversationError; - - return aiResponseData; - }, - onError: (error) => { - toast.error(error.message || 'Unknown error'); - setWaitingForResponse(false); - }, - onSuccess: (aiResponse) => { - setMessages((currentMessages) => { - return [...currentMessages, aiResponse.msg as Message]; + + // Invoke the function and get the response as a ReadableStream + const url = `${supabaseUrl}/functions/v1/chat`; + const headers = { + 'Authorization': `Bearer ${supabaseToken}`, + 'Content-Type': 'application/json', + }; + const response = await fetch(url, { + method: 'POST', + headers: headers, + body: JSON.stringify({ + messageHistory: [...messages, userMessage], + timestamp: new Date().toISOString(), + }), }); + console.log('Response from chat function:', response); + + if (!response.ok) { + throw new Error('Network response was not ok'); + } + + if (response.body) { + const reader = response.body.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunks = new TextDecoder().decode(value).split('\n').filter(Boolean); + for (const chunk of chunks) { + if (chunk) { + const aiResponseData = JSON.parse(chunk); + console.log('Chunk from AI:', aiResponseData); + } + } + } + } catch (error) { + console.error('Stream reading failed', error); + } finally { + reader.releaseLock(); + } + } setWaitingForResponse(false); }, }); From decb57802ee948970205c03138d6aebeb521141c Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Thu, 29 Feb 2024 17:20:35 -0700 Subject: [PATCH 08/14] Streamed Response --- app/src/components/.ChatLog.tsx.swp | Bin 0 -> 12288 bytes app/src/components/Chat.tsx | 45 ++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 app/src/components/.ChatLog.tsx.swp diff --git a/app/src/components/.ChatLog.tsx.swp b/app/src/components/.ChatLog.tsx.swp new file mode 100644 index 0000000000000000000000000000000000000000..2fa1dd101fd54da47daa265a262a588667a1ecf6 GIT binary patch literal 12288 zcmeI2&u`;I6vwAMu)tD4>}3%WdSpRKM4YtUUPzOcvK2oBg^C{wtEEU2XWV+%zvLNj zQ(1x2A~+)sdqf~1B=`%EkoLrp1L6i;KpYT%1L8Zj6Q@a*vVsdRmOimPe(!naJ8zy< zb$ZJepWC4)n#&Bw{fwP^^gD6p<`0XN?FA+ecoKw;BZ3~c4@9g@<{htSJ8A83+w$GO zZF-^2y*Y+&PX}ym3wc$AQEN92uLxVo*4DO2WQ#{pOU8D~4*e($M1Thu4!C+D+;1v5 zm=oDF83l|2Co8bT&abXyOZ{Bs5jyk0YbU!H3!{Khz$jo8FbWt2i~>dhqkvK1Kcj$( z7T7x&&0;a4XN&W~)Va7dC!>H-z$jo8FbWt2i~>dhqkvJsC}0#Y3K#|cg$nRKW2a6r zws|KykN^LV{{H{^KE^%)AA|S600>}%9(WS0frr6^U=jRuFJnJ~Z^0Mf21vm!*aVM( z)8OYN#y$rhg6rTA?13$C7JPXR?!oimA@JL2#y$f#z#iBFO+euHyBYfhybIm{kAq+D zV(fcx6MPCj0M|hR-U64v>);|d3zong;2UW075E6e53YeexCCAT&w!_Z)@lQo!zf@B zFbWt2i~|2t1%~U4sIGsAI<1ZDOf;+yi>vWNZJ>IQu=7?yEjp^`YD8Y}#LAtm1zO&NKvs#V_%s`aMNqgrhoxItNrI(0WF z&D=UV+&GXO22#<}VeE6IME_MWP}HSbN=0ZIwrPCRro6Wpcpi`Gevf6@3S2}HyRncW zu*Im`b!tFUsWZ2ctW`Sl77X;}7BzlVtDgE|&tr9nE`2G$xyN7i}5 zZqt5vXi2}qvK|*dol?Dt_VkdWi@`o-{AD4r>r@Kb94C}^txqJizVEc=>HjBl1DE6Z zRHRZjwJ~Xzt4KM8y{^^0;q>GYO-~=u-=4tRoI=8axt&Qw6<}JX>7z~6e`HdtY&x>O z-^`*%F&12WbW+B|x6GJ%!ShLqLr+xqsN1D#B1JrFSSD{?R?1$^PrTW=ou?aAn|)O_ zT*ngwy(&;h$W_9ZmI(TEC6UT?j`a1c-(MQZt;aD@fmt~^WH;8sL_=Dnx6CWAe(7Y< z9o1_1g&vh}t3*9(tuo5@aggz1>`YbDGA$kMNs)aN##J2kSkJ=j+HO-X#7y!)f4W8; zZPQNv-MGBB&UAgB>)M#`XfQE|!dOvXI6UzbIZ0saqDdF>{pz)pW#58gZI3<_`mH*X z%KIXYixLn!+!mv*pcBl+E_l2{hs;X56Nf%ko7tXt-B4X0nWQwOPJ{AlhYFs1IWg*L JY~`(R`~wdea8v*Q literal 0 HcmV?d00001 diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index f9c9037..3680a14 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -28,13 +28,6 @@ export default function Chat({ const sendMessageAndReceiveResponse = useMutation({ mutationFn: async (userMessage: Message) => { - const { data: sendMessageData, error: sendMessageError } = - await supabaseClient - .from('conversations') - .update({ context: [...messages, userMessage] }) - .eq('id', conversationId); - - if (sendMessageError) throw sendMessageError; setMessages([...messages, userMessage]); setWaitingForResponse(true); @@ -53,8 +46,6 @@ export default function Chat({ timestamp: new Date().toISOString(), }), }); - - console.log('Response from chat function:', response); if (!response.ok) { throw new Error('Network response was not ok'); @@ -64,19 +55,49 @@ export default function Chat({ const reader = response.body.getReader(); try { + let completeResponse = ''; while (true) { const { done, value } = await reader.read(); if (done) break; - const chunks = new TextDecoder().decode(value).split('\n').filter(Boolean); + const chunks = new TextDecoder().decode(value).split('\n'); for (const chunk of chunks) { if (chunk) { - const aiResponseData = JSON.parse(chunk); - console.log('Chunk from AI:', aiResponseData); + const aiResponse = JSON.parse(chunk); + if (aiResponse.message) { + + completeResponse += aiResponse.message; + setMessages((prevMessages) => { + const updatedMessages = [...prevMessages]; + const lastMessageIndex = updatedMessages.length - 1; + if (lastMessageIndex >= 0 && updatedMessages[lastMessageIndex].role === 'assistant') { + // If the last message is from the assistant, update its content + updatedMessages[lastMessageIndex] = { + ...updatedMessages[lastMessageIndex], + content: updatedMessages[lastMessageIndex].content + aiResponse.message, + }; + } else { + // Otherwise, add a new message from the assistant + updatedMessages.push({ role: 'assistant', content: aiResponse.message }); + } + return updatedMessages; + }); + } } } + setWaitingForResponse(false); + + const updatedContext = [...messages, userMessage, { role: 'assistant', content: completeResponse }]; + const updateResponse = await supabaseClient + .from('conversations') + .update({ context: updatedContext }) + .eq('id', conversationId); + if (updateResponse.error) { + throw updateResponse.error; + } } } catch (error) { console.error('Stream reading failed', error); + setWaitingForResponse(false); } finally { reader.releaseLock(); } From 43fb39d1b0d0204993ac0a9fde7e3cb43efe9a5a Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Thu, 29 Feb 2024 18:21:54 -0700 Subject: [PATCH 09/14] fixed an oversight --- supabase/functions/chat/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index e80cb63..00f1140 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -185,7 +185,7 @@ const chat = async (req: Request) => { ); for await (const chunk of responseMessageGenerator) { - const jsonResponse = JSON.stringify({ token: chunk }) + "\n"; + const jsonResponse = JSON.stringify({ message: chunk }) + "\n"; const encodedChunk = new TextEncoder().encode(jsonResponse); controller.enqueue(encodedChunk); } From af01c899cc5dc1aada07eded911ae345edabddff Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Mon, 4 Mar 2024 09:21:24 -0700 Subject: [PATCH 10/14] fix stream bug --- app/src/components/.ChatLog.tsx.swp | Bin 12288 -> 0 bytes app/src/components/Chat.tsx | 1 - 2 files changed, 1 deletion(-) delete mode 100644 app/src/components/.ChatLog.tsx.swp diff --git a/app/src/components/.ChatLog.tsx.swp b/app/src/components/.ChatLog.tsx.swp deleted file mode 100644 index 2fa1dd101fd54da47daa265a262a588667a1ecf6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&u`;I6vwAMu)tD4>}3%WdSpRKM4YtUUPzOcvK2oBg^C{wtEEU2XWV+%zvLNj zQ(1x2A~+)sdqf~1B=`%EkoLrp1L6i;KpYT%1L8Zj6Q@a*vVsdRmOimPe(!naJ8zy< zb$ZJepWC4)n#&Bw{fwP^^gD6p<`0XN?FA+ecoKw;BZ3~c4@9g@<{htSJ8A83+w$GO zZF-^2y*Y+&PX}ym3wc$AQEN92uLxVo*4DO2WQ#{pOU8D~4*e($M1Thu4!C+D+;1v5 zm=oDF83l|2Co8bT&abXyOZ{Bs5jyk0YbU!H3!{Khz$jo8FbWt2i~>dhqkvK1Kcj$( z7T7x&&0;a4XN&W~)Va7dC!>H-z$jo8FbWt2i~>dhqkvJsC}0#Y3K#|cg$nRKW2a6r zws|KykN^LV{{H{^KE^%)AA|S600>}%9(WS0frr6^U=jRuFJnJ~Z^0Mf21vm!*aVM( z)8OYN#y$rhg6rTA?13$C7JPXR?!oimA@JL2#y$f#z#iBFO+euHyBYfhybIm{kAq+D zV(fcx6MPCj0M|hR-U64v>);|d3zong;2UW075E6e53YeexCCAT&w!_Z)@lQo!zf@B zFbWt2i~|2t1%~U4sIGsAI<1ZDOf;+yi>vWNZJ>IQu=7?yEjp^`YD8Y}#LAtm1zO&NKvs#V_%s`aMNqgrhoxItNrI(0WF z&D=UV+&GXO22#<}VeE6IME_MWP}HSbN=0ZIwrPCRro6Wpcpi`Gevf6@3S2}HyRncW zu*Im`b!tFUsWZ2ctW`Sl77X;}7BzlVtDgE|&tr9nE`2G$xyN7i}5 zZqt5vXi2}qvK|*dol?Dt_VkdWi@`o-{AD4r>r@Kb94C}^txqJizVEc=>HjBl1DE6Z zRHRZjwJ~Xzt4KM8y{^^0;q>GYO-~=u-=4tRoI=8axt&Qw6<}JX>7z~6e`HdtY&x>O z-^`*%F&12WbW+B|x6GJ%!ShLqLr+xqsN1D#B1JrFSSD{?R?1$^PrTW=ou?aAn|)O_ zT*ngwy(&;h$W_9ZmI(TEC6UT?j`a1c-(MQZt;aD@fmt~^WH;8sL_=Dnx6CWAe(7Y< z9o1_1g&vh}t3*9(tuo5@aggz1>`YbDGA$kMNs)aN##J2kSkJ=j+HO-X#7y!)f4W8; zZPQNv-MGBB&UAgB>)M#`XfQE|!dOvXI6UzbIZ0saqDdF>{pz)pW#58gZI3<_`mH*X z%KIXYixLn!+!mv*pcBl+E_l2{hs;X56Nf%ko7tXt-B4X0nWQwOPJ{AlhYFs1IWg*L JY~`(R`~wdea8v*Q diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index 3680a14..47e35b3 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -84,7 +84,6 @@ export default function Chat({ } } } - setWaitingForResponse(false); const updatedContext = [...messages, userMessage, { role: 'assistant', content: completeResponse }]; const updateResponse = await supabaseClient From 658d83da9d8377ba3b33e15416714123e7ee98c9 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Wed, 6 Mar 2024 11:47:52 -0700 Subject: [PATCH 11/14] removed unused code that I forgot was there --- supabase/functions/chat/index.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index 00f1140..c806558 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -87,20 +87,6 @@ async function getRelevantRecords( const messageHistory = msgData.messageHistory; const timestamp = msgData.timestamp; - let lastMessage = [messageHistory[messageHistory.length - 1]]; - - const systemMessage = { - role: "system", - content: `Your objective is to determine the intent of the users message. Their requests can vary but will often be asking about their day, week, month and life. - Use the current date time ${timestamp} to help you answer their questions. - `, - }; - - const optimnizedUserMsg = await generateResponse( - useOpenRouter, - useOllama, - lastMessage - ); console.log(timestamp); // Embed the last messageHistory message using OpenAI's embeddings API const embeddingsResponse = await openaiClient.embeddings.create({ From 6732fca20d0c8943b2fc95bf90c766df28034f6b Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Wed, 6 Mar 2024 12:06:04 -0700 Subject: [PATCH 12/14] created new flag --- supabase/functions/chat/index.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/supabase/functions/chat/index.ts b/supabase/functions/chat/index.ts index c806558..16012da 100644 --- a/supabase/functions/chat/index.ts +++ b/supabase/functions/chat/index.ts @@ -81,13 +81,10 @@ async function* generateResponse( async function getRelevantRecords( openaiClient: ChatClient, supabase: any, - msgData: any, + messageHistory: Message[], ): Promise { - const messageHistory = msgData.messageHistory; - const timestamp = msgData.timestamp; - - console.log(timestamp); + // Embed the last messageHistory message using OpenAI's embeddings API const embeddingsResponse = await openaiClient.embeddings.create({ model: "text-embedding-3-small", @@ -144,6 +141,7 @@ const chat = async (req: Request) => { const msgData = requestBody as { messageHistory: Message[]; timestamp: string }; if (!msgData.messageHistory) throw new UserError("Missing query in request data"); + const messageHistory = msgData.messageHistory; const relevantRecords = await getRelevantRecords(openaiClient, supabase, messageHistory) From 7510b173d07efdbade49536cada532b77c7f35e1 Mon Sep 17 00:00:00 2001 From: Shaun Offenbacher Date: Wed, 6 Mar 2024 12:06:29 -0700 Subject: [PATCH 13/14] created new flag --- app/src/components/Chat.tsx | 10 +++++++--- app/src/components/PromptForm.tsx | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/src/components/Chat.tsx b/app/src/components/Chat.tsx index 47e35b3..8ccce85 100644 --- a/app/src/components/Chat.tsx +++ b/app/src/components/Chat.tsx @@ -23,6 +23,7 @@ export default function Chat({ const [messages, setMessages] = useState([]); const [conversationId, setConversationId] = useState(null); const [waitingForResponse, setWaitingForResponse] = useState(false); + const [isStreaming, setIsStreaming] = useState(false); const { supabaseUrl, supabaseToken } = useSupabaseConfig(); @@ -53,6 +54,8 @@ export default function Chat({ if (response.body) { const reader = response.body.getReader(); + setWaitingForResponse(false); + setIsStreaming(true); try { let completeResponse = ''; @@ -68,7 +71,7 @@ export default function Chat({ completeResponse += aiResponse.message; setMessages((prevMessages) => { const updatedMessages = [...prevMessages]; - const lastMessageIndex = updatedMessages.length - 1; + const lastMessageIndex = updatedMessages.length - 1; if (lastMessageIndex >= 0 && updatedMessages[lastMessageIndex].role === 'assistant') { // If the last message is from the assistant, update its content updatedMessages[lastMessageIndex] = { @@ -96,12 +99,13 @@ export default function Chat({ } } catch (error) { console.error('Stream reading failed', error); + setIsStreaming(false); setWaitingForResponse(false); } finally { reader.releaseLock(); } } - setWaitingForResponse(false); + setIsStreaming(false); }, }); @@ -212,7 +216,7 @@ export default function Chat({ textareaRef={textareaRef} entryData={entryData} setEntryData={setEntryData} - waitingForResponse={waitingForResponse} + isStreaming={isStreaming} sendMessage={() => { if (!entryData.trim()) return; const userMessage = { role: 'user', content: entryData.trim() }; diff --git a/app/src/components/PromptForm.tsx b/app/src/components/PromptForm.tsx index 6cb529c..2ca9fd1 100644 --- a/app/src/components/PromptForm.tsx +++ b/app/src/components/PromptForm.tsx @@ -6,13 +6,13 @@ export default function PromptForm({ textareaRef, entryData, setEntryData, - waitingForResponse, + isStreaming, sendMessage, }: { textareaRef: React.RefObject; entryData: string; setEntryData: React.Dispatch>; - waitingForResponse: boolean; + isStreaming: boolean; sendMessage: () => void; }) { return ( @@ -35,13 +35,13 @@ export default function PromptForm({ textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`; } }} - disabled={waitingForResponse} + disabled={isStreaming} placeholder="What is on your mind?" >