Merge branch 'lectrician1-add-vision'

ztjhz · Jun 3, 2024 · 69fe944 · 69fe944
2 parents 53aeb01 + 202b2f3
commit 69fe944
Show file tree

Hide file tree

Showing 14 changed files with 376 additions and 120 deletions.
diff --git a/src/api/api.ts b/src/api/api.ts
@@ -1,7 +1,10 @@
+import {modelTypes} from '@constants/chat';
 import { ShareGPTSubmitBodyInterface } from '@type/api';
-import { ConfigInterface, MessageInterface, ModelOptions } from '@type/chat';
+import { ConfigInterface, ImageContentInterface, MessageInterface, MessageInterfaceText, ModelOptions, TextContentInterface } from '@type/chat';
 import { isAzureEndpoint } from '@utils/api';
 
+
+
 export const getChatCompletion = async (
   endpoint: string,
   messages: MessageInterface[],
@@ -43,14 +46,36 @@ export const getChatCompletion = async (
     }
   }
 
+  let body: BodyInit;
+
+  // make content: string if model support only text
+  if (modelTypes[config.model] == 'text') {
+      // Convert messages to MessageInterfaceText
+      const textMessages: MessageInterfaceText[] = messages.map(message => {
+          // Find the first text and ignore images
+          const textContent = message.content.find(content => content.type === 'text') as TextContentInterface;
+          return {
+              role: message.role,
+              content: textContent?.text || ''
+          };
+      });
+      body = JSON.stringify({
+          messages: textMessages,
+          ...config,
+          max_tokens: undefined,
+      });
+  } else {
+      body = JSON.stringify({
+          messages,
+          ...config,
+          max_tokens: undefined,
+      });
+  }
+
   const response = await fetch(endpoint, {
     method: 'POST',
     headers,
-    body: JSON.stringify({
-      messages,
-      ...config,
-      max_tokens: undefined,
-    }),
+    body: body,
   });
   if (!response.ok) throw new Error(await response.text());
 
@@ -96,23 +121,46 @@ export const getChatCompletionStream = async (
     }
   }
 
+  let body: BodyInit;
+
+  // make content: string if model support only text
+  if (modelTypes[config.model] == 'text') {
+      // Convert messages to MessageInterfaceText
+      const textMessages: MessageInterfaceText[] = messages.map(message => {
+          // Find the first text and ignore images
+          const textContent = message.content.find(content => content.type === 'text') as TextContentInterface;
+          return {
+              role: message.role,
+              content: textContent?.text || ''
+          };
+      });
+      body = JSON.stringify({
+          messages: textMessages,
+          ...config,
+          max_tokens: config.max_tokens,
+          stream: true,
+      });
+  } else {
+      body = JSON.stringify({
+          messages,
+          ...config,
+          max_tokens: config.max_tokens,
+          stream: true,
+      });
+  }
+
   const response = await fetch(endpoint, {
     method: 'POST',
     headers,
-    body: JSON.stringify({
-      messages,
-      ...config,
-      max_tokens: undefined,
-      stream: true,
-    }),
+    body: body,
   });
   if (response.status === 404 || response.status === 405) {
     const text = await response.text();
 
     if (text.includes('model_not_found')) {
       throw new Error(
         text +
-          '\nMessage from Better ChatGPT:\nPlease ensure that you have access to the GPT-4 API!'
+        '\nMessage from Better ChatGPT:\nPlease ensure that you have access to the GPT-4 API!'
       );
     } else {
       throw new Error(

diff --git a/src/assets/icons/FolderIcon.tsx b/src/assets/icons/FolderIcon.tsx
@@ -5,8 +5,8 @@ const FolderIcon = (props: React.SVGProps<SVGSVGElement>) => {
     <svg
       viewBox='0 0 1024 1024'
       fill='currentColor'
-      height='1em'
-      width='1em'
+      height='1.7em'
+      width='1.7em'
       {...props}
     >
       <path d='M880 298.4H521L403.7 186.2a8.15 8.15 0 00-5.5-2.2H144c-17.7 0-32 14.3-32 32v592c0 17.7 14.3 32 32 32h736c17.7 0 32-14.3 32-32V330.4c0-17.7-14.3-32-32-32zM840 768H184V256h188.5l119.6 114.4H840V768z' />

diff --git a/src/components/Chat/ChatContent/ChatContent.tsx b/src/components/Chat/ChatContent/ChatContent.tsx
@@ -12,6 +12,7 @@ import useSubmit from '@hooks/useSubmit';
 import DownloadChat from './DownloadChat';
 import CloneChat from './CloneChat';
 import ShareGPT from '@components/ShareGPT';
+import { ImageContentInterface, TextContentInterface } from '@type/chat';
 
 const ChatContent = () => {
   const inputRole = useStore((state) => state.inputRole);
@@ -79,7 +80,10 @@ const ChatContent = () => {
 
           <Message
             role={inputRole}
-            content=''
+            // For now we always initizlize a new message with an empty text content.
+            // It is possible to send a message to the API without a TextContentInterface, 
+            // but the UI would need to be modified to allow the user to control the order of text and image content
+            content={[{type: 'text', text: ''} as TextContentInterface]}
             messageIndex={stickyIndex}
             sticky
           />

diff --git a/src/components/Chat/ChatContent/Message/CommandPrompt/CommandPrompt.tsx b/src/components/Chat/ChatContent/Message/CommandPrompt/CommandPrompt.tsx
@@ -6,11 +6,12 @@ import { matchSorter } from 'match-sorter';
 import { Prompt } from '@type/prompt';
 
 import useHideOnOutsideClick from '@hooks/useHideOnOutsideClick';
+import { ContentInterface } from '@type/chat';
 
 const CommandPrompt = ({
   _setContent,
 }: {
-  _setContent: React.Dispatch<React.SetStateAction<string>>;
+  _setContent: React.Dispatch<React.SetStateAction<ContentInterface[]>>;
 }) => {
   const { t } = useTranslation();
   const prompts = useStore((state) => state.prompts);
@@ -69,7 +70,7 @@ const CommandPrompt = ({
             <li
               className='px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white cursor-pointer text-start w-full'
               onClick={() => {
-                _setContent((prev) => prev + cp.prompt);
+                _setContent((prev) => [{type: 'text', text: prev + cp.prompt}, ...prev.slice(1)]);
                 setDropDown(false);
               }}
               key={cp.id}

diff --git a/src/components/Chat/ChatContent/Message/Message.tsx b/src/components/Chat/ChatContent/Message/Message.tsx
@@ -4,7 +4,7 @@ import useStore from '@store/store';
 import Avatar from './Avatar';
 import MessageContent from './MessageContent';
 
-import { Role } from '@type/chat';
+import { ContentInterface, Role } from '@type/chat';
 import RoleSelector from './RoleSelector';
 
 // const backgroundStyle: { [role in Role]: string } = {
@@ -22,7 +22,7 @@ const Message = React.memo(
     sticky = false,
   }: {
     role: Role;
-    content: string;
+    content: ContentInterface[],
     messageIndex: number;
     sticky?: boolean;
   }) => {

diff --git a/src/components/Chat/ChatContent/Message/MessageContent.tsx b/src/components/Chat/ChatContent/Message/MessageContent.tsx
@@ -3,6 +3,7 @@ import useStore from '@store/store';
 
 import ContentView from './View/ContentView';
 import EditView from './View/EditView';
+import { ContentInterface } from '@type/chat';
 
 const MessageContent = ({
   role,
@@ -11,7 +12,7 @@ const MessageContent = ({
   sticky = false,
 }: {
   role: string;
-  content: string;
+  content: ContentInterface[];
   messageIndex: number;
   sticky?: boolean;
 }) => {

diff --git a/src/components/Chat/ChatContent/Message/View/ContentView.tsx b/src/components/Chat/ChatContent/Message/View/ContentView.tsx
@@ -19,7 +19,7 @@ import CrossIcon from '@icon/CrossIcon';
 
 import useSubmit from '@hooks/useSubmit';
 
-import { ChatInterface } from '@type/chat';
+import { ChatInterface, ContentInterface, ImageContentInterface, TextContentInterface } from '@type/chat';
 
 import { codeLanguageSubset } from '@constants/chat';
 
@@ -41,7 +41,7 @@ const ContentView = memo(
     messageIndex,
   }: {
     role: string;
-    content: string;
+    content: ContentInterface[],
     setIsEdit: React.Dispatch<React.SetStateAction<boolean>>;
     messageIndex: number;
   }) => {
@@ -100,7 +100,7 @@ const ContentView = memo(
     };
 
     const handleCopy = () => {
-      navigator.clipboard.writeText(content);
+      navigator.clipboard.writeText((content[0] as TextContentInterface).text);
     };
 
     return (
@@ -129,12 +129,19 @@ const ContentView = memo(
                 p,
               }}
             >
-              {content}
+              {(content[0] as TextContentInterface).text}
             </ReactMarkdown>
           ) : (
-            <span className='whitespace-pre-wrap'>{content}</span>
+            <span className='whitespace-pre-wrap'>{(content[0] as TextContentInterface).text}</span>
           )}
         </div>
+        <div className="flex gap-4">
+          {(content.slice(1) as ImageContentInterface[]).map((image, index) => (
+            <div key={index} className="image-container">
+              <img src={image.image_url.url} alt={`uploaded-${index}`} className="h-20" />
+            </div>
+          ))}
+        </div>
         <div className='flex justify-end gap-2 w-full mt-2'>
           {isDelete || (
             <>