From 157769eeb4439fccb6788c2e307ea661a8127f18 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 12 Jan 2024 14:35:23 +0000 Subject: [PATCH 1/4] Add API endpoint for searching documents --- application/api/answer/routes.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 2d2019ef7..13685154e 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -343,3 +343,32 @@ def api_answer(): traceback.print_exc() print(str(e)) return bad_request(500, str(e)) + + +@answer.route("/api/search", methods=["POST"]) +def api_search(): + data = request.get_json() + # get parameter from url question + question = data["question"] + + if not embeddings_key_set: + embeddings_key = data["embeddings_key"] + else: + embeddings_key = settings.EMBEDDINGS_KEY + if "active_docs" in data: + vectorstore = get_vectorstore({"active_docs": data["active_docs"]}) + else: + vectorstore = "" + docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key) + + docs = docsearch.search(question, k=2) + + source_log_docs = [] + for doc in docs: + if doc.metadata: + source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content}) + else: + source_log_docs.append({"title": doc.page_content, "text": doc.page_content}) + yield f"data:{data}\n\n" + return source_log_docs + From a04cdee33f21c443b5d96d53041fcb9aebc9475b Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 12 Jan 2024 14:38:15 +0000 Subject: [PATCH 2/4] Refactor source log generation in complete_stream function --- application/api/answer/routes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 13685154e..3fa32a6ea 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -117,12 +117,9 @@ def complete_stream(question, docsearch, chat_history, api_key, prompt_id, conve source_log_docs = [] for doc in docs: if doc.metadata: - data = json.dumps({"type": "source", "doc": doc.page_content[:10], "metadata": doc.metadata}) source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content}) else: - data = json.dumps({"type": "source", "doc": doc.page_content[:10]}) source_log_docs.append({"title": doc.page_content, "text": doc.page_content}) - yield f"data:{data}\n\n" if len(chat_history) > 1: tokens_current_history = 0 From 6a70e3e45b5c57c24bc38ab2d534705e6365ec72 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 12 Jan 2024 14:39:17 +0000 Subject: [PATCH 3/4] Commented out unused code in api_search function --- application/api/answer/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 3fa32a6ea..8c661e0bd 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -366,6 +366,6 @@ def api_search(): source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content}) else: source_log_docs.append({"title": doc.page_content, "text": doc.page_content}) - yield f"data:{data}\n\n" + #yield f"data:{data}\n\n" return source_log_docs From f4f056449f6b6ca87c012223828bd6f982b98637 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Mon, 15 Jan 2024 20:23:18 +0530 Subject: [PATCH 4/4] integrate /api/search endpoint, get sources post stream --- frontend/src/conversation/conversationApi.ts | 78 +++++++++++++++---- .../src/conversation/conversationSlice.ts | 39 +++++----- 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/frontend/src/conversation/conversationApi.ts b/frontend/src/conversation/conversationApi.ts index 542a3a99a..a01d034a2 100644 --- a/frontend/src/conversation/conversationApi.ts +++ b/frontend/src/conversation/conversationApi.ts @@ -12,20 +12,20 @@ export function fetchAnswerApi( promptId: string | null, ): Promise< | { - result: any; - answer: any; - sources: any; - conversationId: any; - query: string; - } + result: any; + answer: any; + sources: any; + conversationId: any; + query: string; + } | { - result: any; - answer: any; - sources: any; - query: string; - conversationId: any; - title: any; - } + result: any; + answer: any; + sources: any; + query: string; + conversationId: any; + title: any; + } > { let namePath = selectedDocs.name; if (selectedDocs.language === namePath) { @@ -128,7 +128,6 @@ export function fetchAnswerSteaming( conversation_id: conversationId, prompt_id: promptId, }; - fetch(apiHost + '/stream', { method: 'POST', headers: { @@ -183,7 +182,58 @@ export function fetchAnswerSteaming( }); }); } +export function searchEndpoint( + question: string, + apiKey: string, + selectedDocs: Doc, + conversation_id: string | null, + history: Array = [], +) { + /* + "active_docs": "default", + "question": "Summarise", + "conversation_id": null, + "history": "[]" */ + let namePath = selectedDocs.name; + if (selectedDocs.language === namePath) { + namePath = '.project'; + } + + let docPath = 'default'; + if (selectedDocs.location === 'local') { + docPath = 'local' + '/' + selectedDocs.name + '/'; + } else if (selectedDocs.location === 'remote') { + docPath = + selectedDocs.language + + '/' + + namePath + + '/' + + selectedDocs.version + + '/' + + selectedDocs.model + + '/'; + } + const body = { + question: question, + active_docs: docPath, + conversation_id, + history + }; + return fetch(`${apiHost}/api/search`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify( + body + ), + }).then((response) => response.json()) + .then((data) => { + return data; + }) + .catch(err => console.log(err)) +} export function sendFeedback( prompt: string, response: string, diff --git a/frontend/src/conversation/conversationSlice.ts b/frontend/src/conversation/conversationSlice.ts index ca92092b5..3f840d3c1 100644 --- a/frontend/src/conversation/conversationSlice.ts +++ b/frontend/src/conversation/conversationSlice.ts @@ -1,6 +1,7 @@ import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit'; import store from '../store'; import { fetchAnswerApi, fetchAnswerSteaming } from './conversationApi'; +import { searchEndpoint } from './conversationApi'; import { Answer, ConversationState, Query, Status } from './conversationModels'; import { getConversations } from '../preferences/preferenceApi'; import { setConversations } from '../preferences/preferenceSlice'; @@ -29,6 +30,7 @@ export const fetchAnswer = createAsyncThunk( (event) => { const data = JSON.parse(event.data); + // check if the 'end' event has been received if (data.type === 'end') { // set status to 'idle' @@ -40,24 +42,22 @@ export const fetchAnswer = createAsyncThunk( .catch((error) => { console.error('Failed to fetch conversations: ', error); }); - } else if (data.type === 'source') { - // check if data.metadata exists - let result; - if (data.metadata && data.metadata.title) { - const titleParts = data.metadata.title.split('/'); - result = { - title: titleParts[titleParts.length - 1], - text: data.doc, - }; - } else { - result = { title: data.doc, text: data.doc }; - } - dispatch( - updateStreamingSource({ - index: state.conversation.queries.length - 1, - query: { sources: [result] }, - }), - ); + + searchEndpoint( //search for sources post streaming + question, + state.preference.apiKey, + state.preference.selectedDocs!, + state.conversation.conversationId, + state.conversation.queries + ).then(sources => { + //dispatch streaming sources + dispatch( + updateStreamingSource({ + index: state.conversation.queries.length - 1, + query: { sources }, + }), + ); + }); } else if (data.type === 'id') { dispatch( updateConversationId({ @@ -165,9 +165,10 @@ export const conversationSlice = createSlice({ state, action: PayloadAction<{ index: number; query: Partial }>, ) { + const { index, query } = action.payload; if (!state.queries[index].sources) { - state.queries[index].sources = [query.sources![0]]; + state.queries[index].sources = query?.sources; } else { state.queries[index].sources!.push(query.sources![0]); }