switch message data structure over to mimicing the data structure of …

…the openai request

src/api/api.ts

@@ -3,17 +3,6 @@ import { ConfigInterface, ImageContentInterface, MessageInterface, ModelOptions
 import { isAzureEndpoint } from '@utils/api';
 
 
-// convert message blob urls to base64
-const blobToBase64 = async (blob: Blob) => {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader();
-    reader.onerror = reject;
-    reader.onload = () => {
-      resolve(reader.result);
-    };
-    reader.readAsDataURL(blob);
-  });
-};
 
 export const getChatCompletion = async (
   endpoint: string,
@@ -54,37 +43,11 @@ export const getChatCompletion = async (
     }
   }
 
-  // do this for all the messages that have image content
-  const validMessages = await Promise.all(
-    messages.map(async (message) => {
-      return {
-        ...message,
-        content: message.content.map(async (content) => {
-          if ('image_url' in content) {
-            let imageContent = content as ImageContentInterface
-            if (imageContent.image_url.url.startsWith('blob:')) {
-              const blob = await fetch(imageContent.image_url.url).then((r) => r.blob());
-              let base64 = await blobToBase64(blob)
-              let url = `data:image/jpeg;base64,${base64}`
-              console.log(url)
-              return {
-                ...imageContent,
-                image_url: {
-                  url: url
-                }
-              }
-            }
-            return message.content;
-          }
-        })
-      }
-    }))
-
   const response = await fetch(endpoint, {
     method: 'POST',
     headers,
     body: JSON.stringify({
-      validImages: validMessages,
+      messages,
       ...config,
       max_tokens: undefined,
     }),
@@ -133,39 +96,11 @@ export const getChatCompletionStream = async (
     }
   }
 
-  const validMessages = await Promise.all(
-    messages.map(async (message) => {
-      return {
-        ...message,
-        content: await Promise.all(message.content.map(async (content) => {
-          if ('image_url' in content) {
-            let imageContent = content as ImageContentInterface
-            if (imageContent.image_url.url.startsWith('blob:')) {
-              const blob = await fetch(imageContent.image_url.url).then((r) => r.blob());
-              let base64 = await blobToBase64(blob)
-              let url = base64
-              console.log(url)
-              return {
-                type: 'image_url',
-                image_url: {
-                  url: url
-                }
-              }
-            }
-            // if not a locally stored image (future we can allow for urls from internet)
-            return content;
-          }
-          // if text and not image_url
-          return content;
-        }))
-      }
-    }))
-
   const response = await fetch(endpoint, {
     method: 'POST',
     headers,
     body: JSON.stringify({
-      messages: validMessages,
+      messages,
       ...config,
       max_tokens: config.max_tokens,
       stream: true,

src/components/Chat/ChatContent/ChatContent.tsx

@@ -69,8 +69,7 @@ const ChatContent = () => {
                 <React.Fragment key={index}>
                   <Message
                     role={message.role}
-                    text={(message.content[0] as TextContentInterface).text}
-                    image_urls={message.content[1] ? (message.content.slice(1) as ImageContentInterface[]).map((imageContent) => imageContent.image_url.url) : []}
+                    content={message.content}
                     messageIndex={index}
                   />
                   {!generating && advancedMode && <NewMessageButton messageIndex={index} />}
@@ -81,8 +80,10 @@ const ChatContent = () => {
 
           <Message
             role={inputRole}
-            text=''
-            image_urls={[]}
+            // For now we always initizlize a new message with an empty text content.
+            // It is possible to send a message to the API without a TextContentInterface, 
+            // but the UI would need to be modified to allow the user to control the order of text and image content
+            content={[{type: 'text', text: ''} as TextContentInterface]}
             messageIndex={stickyIndex}
             sticky
           />

src/components/Chat/ChatContent/Message/CommandPrompt/CommandPrompt.tsx

@@ -6,11 +6,12 @@ import { matchSorter } from 'match-sorter';
 import { Prompt } from '@type/prompt';
 
 import useHideOnOutsideClick from '@hooks/useHideOnOutsideClick';
+import { ContentInterface } from '@type/chat';
 
 const CommandPrompt = ({
   _setContent,
 }: {
-  _setContent: React.Dispatch<React.SetStateAction<string>>;
+  _setContent: React.Dispatch<React.SetStateAction<ContentInterface[]>>;
 }) => {
   const { t } = useTranslation();
   const prompts = useStore((state) => state.prompts);
@@ -69,7 +70,7 @@ const CommandPrompt = ({
             <li
               className='px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white cursor-pointer text-start w-full'
               onClick={() => {
-                _setContent((prev) => prev + cp.prompt);
+                _setContent((prev) => [{type: 'text', text: prev + cp.prompt}, ...prev.slice(1)]);
                 setDropDown(false);
               }}
               key={cp.id}

src/components/Chat/ChatContent/Message/Message.tsx

@@ -4,7 +4,7 @@ import useStore from '@store/store';
 import Avatar from './Avatar';
 import MessageContent from './MessageContent';
 
-import { Role } from '@type/chat';
+import { ContentInterface, Role } from '@type/chat';
 import RoleSelector from './RoleSelector';
 
 // const backgroundStyle: { [role in Role]: string } = {
@@ -17,14 +17,12 @@ const backgroundStyle = ['dark:bg-gray-800', 'bg-gray-50 dark:bg-gray-650'];
 const Message = React.memo(
   ({
     role,
-    text,
-    image_urls,
+    content,
     messageIndex,
     sticky = false,
   }: {
     role: Role;
-    text: string;
-    image_urls: string[];
+    content: ContentInterface[],
     messageIndex: number;
     sticky?: boolean;
   }) => {
@@ -54,8 +52,7 @@ const Message = React.memo(
               />}
             <MessageContent
               role={role}
-              text={text}
-              image_urls={image_urls}
+              content={content}
               messageIndex={messageIndex}
               sticky={sticky}
             />

src/components/Chat/ChatContent/Message/MessageContent.tsx

@@ -3,17 +3,16 @@ import useStore from '@store/store';
 
 import ContentView from './View/ContentView';
 import EditView from './View/EditView';
+import { ContentInterface } from '@type/chat';
 
 const MessageContent = ({
   role,
-  text,
-  image_urls,
+  content,
   messageIndex,
   sticky = false,
 }: {
   role: string;
-  text: string;
-  image_urls: string[];
+  content: ContentInterface[];
   messageIndex: number;
   sticky?: boolean;
 }) => {
@@ -25,17 +24,15 @@ const MessageContent = ({
       {advancedMode && <div className='flex flex-grow flex-col gap-3'></div>}
       {isEdit ? (
         <EditView
-          text={text}
-          image_urls={image_urls}
+          content={content}
           setIsEdit={setIsEdit}
           messageIndex={messageIndex}
           sticky={sticky}
         />
       ) : (
         <ContentView
           role={role}
-          text={text}
-          image_urls={image_urls}
+          content={content}
           setIsEdit={setIsEdit}
           messageIndex={messageIndex}
         />

src/components/Chat/ChatContent/Message/View/ContentView.tsx

@@ -19,7 +19,7 @@ import CrossIcon from '@icon/CrossIcon';
 
 import useSubmit from '@hooks/useSubmit';
 
-import { ChatInterface } from '@type/chat';
+import { ChatInterface, ContentInterface, ImageContentInterface, TextContentInterface } from '@type/chat';
 
 import { codeLanguageSubset } from '@constants/chat';
 
@@ -36,14 +36,12 @@ import CodeBlock from '../CodeBlock';
 const ContentView = memo(
   ({
     role,
-    text,
-    image_urls,
+    content,
     setIsEdit,
     messageIndex,
   }: {
     role: string;
-    text: string;
-    image_urls: string[];
+    content: ContentInterface[],
     setIsEdit: React.Dispatch<React.SetStateAction<boolean>>;
     messageIndex: number;
   }) => {
@@ -102,7 +100,7 @@ const ContentView = memo(
     };
 
     const handleCopy = () => {
-      navigator.clipboard.writeText(text);
+      navigator.clipboard.writeText((content[0] as TextContentInterface).text);
     };
 
     return (
@@ -131,16 +129,16 @@ const ContentView = memo(
                 p,
               }}
             >
-              {text}
+              {(content[0] as TextContentInterface).text}
             </ReactMarkdown>
           ) : (
-            <span className='whitespace-pre-wrap'>{text}</span>
+            <span className='whitespace-pre-wrap'>{(content[0] as TextContentInterface).text}</span>
           )}
         </div>
-        <div className="flex">
-          {image_urls.map((image, index) => (
+        <div className="flex gap-4">
+          {(content.slice(1) as ImageContentInterface[]).map((image, index) => (
             <div key={index} className="image-container">
-              <img src={image} alt={`uploaded-${index}`} className="mr-2 h-20" />
+              <img src={image.image_url.url} alt={`uploaded-${index}`} className="h-20" />
             </div>
           ))}
         </div>