Merge pull request #117 from danny-avila/search-final

Search final
danny-avila · Mar 23, 2023 · c6fb301 · c6fb301
2 parents d3046dc + 95cf27e
commit c6fb301
Show file tree

Hide file tree

Showing 40 changed files with 989 additions and 363 deletions.
diff --git a/api/.env.example b/api/.env.example
@@ -18,12 +18,47 @@ MONGO_URI="mongodb://127.0.0.1:27017/chatgpt-clone"
 # API key configuration.
 # Leave blank if you don't want them.
 OPENAI_KEY=
-CHATGPT_TOKEN=
 BING_TOKEN=
 
-# User System
+# ChatGPT Browser Client (free but use at your own risk)
+# Access token from https://chat.openai.com/api/auth/session
+# Exposes your access token to a 3rd party
+CHATGPT_TOKEN=
+# If you have access to other models on the official site, you can use them here.
+# Defaults to 'text-davinci-002-render-sha' if left empty.
+# options: gpt-4, text-davinci-002-render, text-davinci-002-render-paid, or text-davinci-002-render-sha
+# You cannot use a model that your account does not have access to. You can check 
+# which ones you have access to by opening DevTools and going to the Network tab. 
+# Refresh the page and look at the response body for https://chat.openai.com/backend-api/models.
+BROWSER_MODEL=
+
+# ENABLING SEARCH MESSAGES/CONVOS
+# Requires installation of free self-hosted Meilisearch or Paid Remote Plan (Remote not tested)
+# The easiest setup for this is through docker-compose, which takes care of it for you.
+# SEARCH=TRUE
+SEARCH=TRUE
+
+# REQUIRED FOR SEARCH: MeiliSearch Host, mainly for api server to connect to the search server.
+# must replace '0.0.0.0' with 'meilisearch' if serving meilisearch with docker-compose
+# MEILI_HOST='http://0.0.0.0:7700' # <-- local/remote
+MEILI_HOST='http://meilisearch:7700' # <-- docker-compose
 
+# REQUIRED FOR SEARCH: MeiliSearch HTTP Address, mainly for docker-compose to expose the search server.
+# must replace '0.0.0.0' with 'meilisearch' if serving meilisearch with docker-compose
+# MEILI_HTTP_ADDR='0.0.0.0:7700'  # <-- local/remote
+MEILI_HTTP_ADDR='meilisearch:7700'  # <-- docker-compose
+
+# REQUIRED FOR SEARCH: In production env., needs a secure key, feel free to generate your own.
+# This master key must be at least 16 bytes, composed of valid UTF-8 characters.
+# Meilisearch will throw an error and refuse to launch if no master key is provided or if it is under 16 bytes, 
+# Meilisearch will suggest a secure autogenerated master key.
+# Using docker, it seems recognized as production so use a secure key.
+# MEILI_MASTER_KEY=                                           # <-- no/insecure key for local/remote
+MEILI_MASTER_KEY=JKMW-hGc7v_D1FkJVdbRSDNFLZcUv3S75yrxXP0SmcU # <-- ready made secure key for docker-compose
+
+
+# User System
 # global enable/disable the sample user system.
 # this is not a ready to use user system.
 # dont't use it, unless you can write your own code.
-ENABLE_USER_SYSTEM=
+ENABLE_USER_SYSTEM=FALSE
diff --git a/api/.prettierrc b/api/.prettierrc
@@ -0,0 +1,22 @@
+{
+  "arrowParens": "avoid",
+  "bracketSpacing": true,
+  "endOfLine": "lf",
+  "htmlWhitespaceSensitivity": "css",
+  "insertPragma": false,
+  "singleAttributePerLine": true,
+  "bracketSameLine": false,
+  "jsxBracketSameLine": false,
+  "jsxSingleQuote": false,
+  "printWidth": 110,
+  "proseWrap": "preserve",
+  "quoteProps": "as-needed",
+  "requirePragma": false,
+  "semi": true,
+  "singleQuote": true,
+  "tabWidth": 2,
+  "trailingComma": "none",
+  "useTabs": false,
+  "vueIndentScriptAndStyle": false,
+  "parser": "babel"
+}
diff --git a/api/app/clients/chatgpt-browser.js b/api/app/clients/chatgpt-browser.js
@@ -1,5 +1,6 @@
 require('dotenv').config();
 const { KeyvFile } = require('keyv-file');
+const set = new Set(["gpt-4", "text-davinci-002-render", "text-davinci-002-render-paid", "text-davinci-002-render-sha"]);
 
 const clientOptions = {
   // Warning: This will expose your access token to a third party. Consider the risks before using this.
@@ -10,6 +11,12 @@ const clientOptions = {
   proxy: process.env.PROXY || null,
 };
 
+// You can check which models you have access to by opening DevTools and going to the Network tab. 
+// Refresh the page and look at the response body for https://chat.openai.com/backend-api/models.
+if (set.has(process.env.BROWSER_MODEL)) {
+  clientOptions.model = process.env.BROWSER_MODEL;
+}
+
 const browserClient = async ({ text, onProgress, convo, abortController }) => {
   const { ChatGPTBrowserClient } = await import('@waylaidwanderer/chatgpt-api');
 

diff --git a/api/app/titleConvo.js b/api/app/titleConvo.js
@@ -1,7 +1,7 @@
 const { Configuration, OpenAIApi } = require('openai');
 const _ = require('lodash');
 
-const proxyEnvToAxiosProxy = (proxyString) => {
+const proxyEnvToAxiosProxy = proxyString => {
   if (!proxyString) return null;
 
   const regex = /^([^:]+):\/\/(?:([^:@]*):?([^:@]*)@)?([^:]+)(?::(\d+))?/;
@@ -18,33 +18,37 @@ const proxyEnvToAxiosProxy = (proxyString) => {
 
 const titleConvo = async ({ model, text, response }) => {
   let title = 'New Chat';
+
+  const request = {
+    model: 'gpt-3.5-turbo',
+    messages: [
+      {
+        role: 'system',
+        content:
+          'You are a title-generator with one job: giving a conversation, detect the language and titling the conversation provided by a user in title case, using the same language.'
+      },
+      {
+        role: 'user',
+        content: `In 5 words or less, summarize the conversation below with a title in title case using the language the user writes in. Don't refer to the participants of the conversation nor the language. Do not include punctuation or quotation marks. Your response should be in title case, exclusively containing the title. Conversation:\n\nUser: "${text}"\n\n${model}: "${JSON.stringify(
+          response?.text
+        )}"\n\nTitle: `
+      }
+    ],
+    temperature: 0,
+    presence_penalty: 0,
+    frequency_penalty: 0
+  };
+
+  // console.log('REQUEST', request);
+
   try {
     const configuration = new Configuration({
       apiKey: process.env.OPENAI_KEY
     });
     const openai = new OpenAIApi(configuration);
-    const completion = await openai.createChatCompletion(
-      {
-        model: 'gpt-3.5-turbo',
-        messages: [
-          {
-            role: 'system',
-            content:
-              'You are a title-generator with one job: giving a conversation, detect the language and titling the conversation provided by a user in title case, using the same language.'
-          },
-          {
-            role: 'user',
-            content: `In 5 words or less, summarize the conversation below with a title in title case using the language the user writes in. Don't refer to the participants of the conversation nor the language. Do not include punctuation or quotation marks. Your response should be in title case, exclusively containing the title. Conversation:\n\nUser: "${text}"\n\n${model}: "${JSON.stringify(
-              response?.text
-            )}"\n\nTitle: `
-          }
-        ],
-        temperature: 0,
-        presence_penalty: 0,
-        frequency_penalty: 0,
-      },
-      { proxy: proxyEnvToAxiosProxy(process.env.PROXY || null) }
-    );
+    const completion = await openai.createChatCompletion(request, {
+      proxy: proxyEnvToAxiosProxy(process.env.PROXY || null)
+    });
 
     //eslint-disable-next-line
     title = completion.data.choices[0].message.content.replace(/["\.]/g, '');

diff --git a/api/lib/db/indexSync.js b/api/lib/db/indexSync.js
@@ -0,0 +1,70 @@
+const mongoose = require('mongoose');
+const Conversation = mongoose.models.Conversation;
+const Message = mongoose.models.Message;
+const { MeiliSearch } = require('meilisearch');
+let currentTimeout = null;
+
+// eslint-disable-next-line no-unused-vars
+async function indexSync(req, res, next) {
+  try {
+    if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY || !process.env.SEARCH) {
+      throw new Error('Meilisearch not configured, search will be disabled.');
+    }
+
+    const client = new MeiliSearch({
+      host: process.env.MEILI_HOST,
+      apiKey: process.env.MEILI_MASTER_KEY
+    });
+
+    const { status } = await client.health();
+    // console.log(`Meilisearch: ${status}`);
+    const result = status === 'available' && !!process.env.SEARCH;
+
+    if (!result) {
+      throw new Error('Meilisearch not available');
+    }
+
+    const messageCount = await Message.countDocuments();
+    const convoCount = await Conversation.countDocuments();
+    const messages = await client.index('messages').getStats();
+    const convos = await client.index('convos').getStats();
+    const messagesIndexed = messages.numberOfDocuments;
+    const convosIndexed = convos.numberOfDocuments;
+
+    console.log(`There are ${messageCount} messages in the database, ${messagesIndexed} indexed`);
+    console.log(`There are ${convoCount} convos in the database, ${convosIndexed} indexed`);
+
+    if (messageCount !== messagesIndexed) {
+      console.log('Messages out of sync, indexing');
+      await Message.syncWithMeili();
+    }
+
+    if (convoCount !== convosIndexed) {
+      console.log('Convos out of sync, indexing');
+      await Conversation.syncWithMeili();
+    }
+  } catch (err) {
+    // console.log('in index sync');
+    if (err.message.includes('not found')) {
+      console.log('Creating indices...');
+      currentTimeout = setTimeout(async () => {
+        try {
+          await Message.syncWithMeili();
+          await Conversation.syncWithMeili();
+        } catch (err) {
+          console.error('Trouble creating indices, try restarting the server.');
+        }
+      }, 750);
+    } else {
+      console.error(err);
+      // res.status(500).json({ error: 'Server error' });
+    }
+  }
+}
+
+process.on('exit', () => {
+  console.log('Clearing sync timeouts before exiting...');
+  clearTimeout(currentTimeout);
+});
+
+module.exports = indexSync;
diff --git a/api/lib/utils/misc.js b/api/lib/utils/misc.js
@@ -0,0 +1,15 @@
+const cleanUpPrimaryKeyValue = (value) => {
+  // For Bing convoId handling
+  return value.replace(/--/g, '-');
+};
+
+function replaceSup(text) {
+  if (!text.includes('<sup>')) return text;
+  const replacedText = text.replace(/<sup>/g, '^').replace(/\s+<\/sup>/g, '^');
+  return replacedText;
+}
+
+module.exports = {
+  cleanUpPrimaryKeyValue,
+  replaceSup
+};
diff --git a/api/models/Config.js b/api/models/Config.js
@@ -49,16 +49,16 @@ const configSchema = mongoose.Schema(
 );
 
 // Instance method
-ConfigSchema.methods.incrementCount = function () {
+configSchema.methods.incrementCount = function () {
   this.startupCounts += 1;
 };
 
 // Static methods
-ConfigSchema.statics.findByTag = async function (tag) {
+configSchema.statics.findByTag = async function (tag) {
   return await this.findOne({ tag });
 };
 
-ConfigSchema.statics.updateByTag = async function (tag, update) {
+configSchema.statics.updateByTag = async function (tag, update) {
   return await this.findOneAndUpdate({ tag }, update, { new: true });
 };
 

diff --git a/api/models/Conversation.js b/api/models/Conversation.js
@@ -1,70 +1,8 @@
-const mongoose = require('mongoose');
-const mongoMeili = require('../lib/db/mongoMeili');
+// const { Conversation } = require('./plugins');
+const Conversation = require('./schema/convoSchema');
+const { cleanUpPrimaryKeyValue } = require('../lib/utils/misc');
 const { getMessages, deleteMessages } = require('./Message');
 
-const convoSchema = mongoose.Schema(
-  {
-    conversationId: {
-      type: String,
-      unique: true,
-      required: true,
-      index: true,
-      meiliIndex: true
-    },
-    parentMessageId: {
-      type: String,
-      required: true
-    },
-    title: {
-      type: String,
-      default: 'New Chat',
-      meiliIndex: true
-    },
-    jailbreakConversationId: {
-      type: String,
-      default: null
-    },
-    conversationSignature: {
-      type: String,
-      default: null
-    },
-    clientId: {
-      type: String
-    },
-    invocationId: {
-      type: String
-    },
-    chatGptLabel: {
-      type: String,
-      default: null
-    },
-    promptPrefix: {
-      type: String,
-      default: null
-    },
-    model: {
-      type: String,
-      required: true
-    },
-    user: {
-      type: String
-    },
-    suggestions: [{ type: String }],
-    messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }]
-  },
-  { timestamps: true }
-);
-
-// convoSchema.plugin(mongoMeili, {
-//   host: process.env.MEILI_HOST,
-//   apiKey: process.env.MEILI_KEY,
-//   indexName: 'convos', // Will get created automatically if it doesn't exist already
-//   primaryKey: 'conversationId'
-// });
-
-const Conversation =
-  mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
-
 const getConvo = async (user, conversationId) => {
   try {
     return await Conversation.findOne({ user, conversationId }).exec();
@@ -143,15 +81,16 @@ module.exports = {
       }
 
       const cache = {};
+      const convoMap = {};
       const promises = [];
       // will handle a syncing solution soon
       const deletedConvoIds = [];
 
-      convoIds.forEach((convo) =>
+      convoIds.forEach(convo =>
         promises.push(
           Conversation.findOne({
             user,
-            conversationId: convo.conversationId
+            conversationId: cleanUpPrimaryKeyValue(convo.conversationId)
           }).exec()
         )
       );
@@ -166,6 +105,7 @@ module.exports = {
             cache[page] = [];
           }
           cache[page].push(convo);
+          convoMap[convo.conversationId] = convo;
           return true;
         }
       });
@@ -183,6 +123,7 @@ module.exports = {
         pageSize,
         // will handle a syncing solution soon
         filter: new Set(deletedConvoIds),
+        convoMap
       };
     } catch (error) {
       console.log(error);
@@ -208,6 +149,7 @@ module.exports = {
   },
   deleteConvos: async (user, filter) => {
     let deleteCount = await Conversation.deleteMany({ ...filter, user }).exec();
+    console.log('deleteCount', deleteCount);
     deleteCount.messages = await deleteMessages(filter);
     return deleteCount;
   }