More prompty updates, test updates

.vscode/launch.json

@@ -36,6 +36,7 @@
         {
             "name": "Tests (Python)",
             "type": "debugpy",
+            "python": "${workspaceFolder}/.venv/bin/python",
             "request": "launch",
             "program": "${file}",
             "purpose": ["debug-test"],

app/backend/approaches/chatapproach.py

@@ -19,36 +19,14 @@ class ChatApproach(Approach, ABC):
     async def run_until_final_call(self, messages, overrides, auth_claims, should_stream) -> tuple:
         pass
 
-    def render_answer_prompt(
-        self,
-        override_prompt: Optional[str],
-        include_follow_up_questions: bool,
-        past_messages: list,
-        user_query: str,
-        sources: str,
-    ) -> RenderedPrompt:
-        if override_prompt is None or override_prompt.startswith(">>>"):
-            injected_prompt = "" if override_prompt is None else override_prompt[3:]
-            return self.prompt_manager.render_prompt(
-                self.answer_prompt,
-                {
-                    "include_follow_up_questions": include_follow_up_questions,
-                    "injected_prompt": injected_prompt,
-                    "past_messages": past_messages,
-                    "user_query": user_query,
-                    "sources": sources,
-                },
-            )
+    def get_system_prompt_variables(self, override_prompt: Optional[str]) -> RenderedPrompt:
+        # Allows client to replace the entire prompt, or to inject into the existing prompt using >>>
+        if override_prompt is None:
+            return {}
+        elif override_prompt.startswith("<<<"):
+            return {"injected_prompt": override_prompt[3:]}
         else:
-            return self.prompt_manager.render_prompt(
-                self.answer_prompt,
-                {
-                    "override_prompt": override_prompt,
-                    "past_messages": past_messages,
-                    "user_query": user_query,
-                    "sources": sources,
-                },
-            )
+            return {"override_prompt": override_prompt}
 
     def get_search_query(self, chat_completion: ChatCompletion, user_query: str):
         response_message = chat_completion.choices[0].message

app/backend/approaches/chatreadretrieveread.py

@@ -149,18 +149,17 @@ async def run_until_final_call(
             minimum_reranker_score,
         )
 
-        sources_content = self.get_sources_content(results, use_semantic_captions, use_image_citation=False)
-        content = "\n".join(sources_content)
-
         # STEP 3: Generate a contextual and content specific answer using the search results and chat history
-
-        # Allow client to replace the entire prompt, or to inject into the existing prompt using >>>
-        rendered_answer_prompt = self.render_answer_prompt(
-            overrides.get("prompt_template"),
-            include_follow_up_questions=bool(overrides.get("suggest_followup_questions")),
-            past_messages=messages[:-1],
-            user_query=original_user_query,
-            sources=content,
+        text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False)
+        rendered_answer_prompt = self.prompt_manager.render_prompt(
+            self.answer_prompt,
+            self.get_system_prompt_variables(overrides.get("prompt_template"))
+            | {
+                "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")),
+                "past_messages": messages[:-1],
+                "user_query": original_user_query,
+                "text_sources": text_sources,
+            },
         )
 
         response_token_limit = 1024
@@ -173,10 +172,8 @@ async def run_until_final_call(
             fallback_to_default=self.ALLOW_NON_GPT_MODELS,
         )
 
-        data_points = {"text": sources_content}
-
         extra_info = {
-            "data_points": data_points,
+            "data_points": {"text": text_sources},
             "thoughts": [
                 ThoughtStep(
                     "Prompt to generate search query",

app/backend/approaches/chatreadretrievereadvision.py

@@ -6,8 +6,6 @@
 from openai.types.chat import (
     ChatCompletion,
     ChatCompletionChunk,
-    ChatCompletionContentPartImageParam,
-    ChatCompletionContentPartParam,
     ChatCompletionMessageParam,
     ChatCompletionToolParam,
 )
@@ -154,51 +152,45 @@ async def run_until_final_call(
             minimum_search_score,
             minimum_reranker_score,
         )
-        sources_content = self.get_sources_content(results, use_semantic_captions, use_image_citation=True)
-        content = "\n".join(sources_content)
 
         # STEP 3: Generate a contextual and content specific answer using the search results and chat history
-
-        # Allow client to replace the entire prompt, or to inject into the existing prompt using >>>
-        rendered_answer_prompt = self.render_answer_prompt(
-            overrides.get("prompt_template"),
-            include_follow_up_questions=bool(overrides.get("suggest_followup_questions")),
-            past_messages=messages[:-1],
-            user_query=original_user_query,
-            sources=content,
-        )
-
-        user_content: list[ChatCompletionContentPartParam] = [
-            {"text": rendered_answer_prompt.new_user_content, "type": "text"}
-        ]
-        image_list: list[ChatCompletionContentPartImageParam] = []
-
+        text_sources = []
+        image_sources = []
         if send_text_to_gptvision:
-            user_content.append({"text": "\n\nSources:\n" + content, "type": "text"})
+            text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=True)
         if send_images_to_gptvision:
             for result in results:
                 url = await fetch_image(self.blob_container_client, result)
                 if url:
-                    image_list.append({"image_url": url, "type": "image_url"})
-            user_content.extend(image_list)
+                    image_sources.append(url)
+
+        rendered_answer_prompt = self.prompt_manager.render_prompt(
+            self.answer_prompt,
+            self.get_system_prompt_variables(overrides.get("prompt_template"))
+            | {
+                "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")),
+                "past_messages": messages[:-1],
+                "user_query": original_user_query,
+                "text_sources": text_sources,
+                "image_sources": image_sources,
+            },
+        )
 
         response_token_limit = 1024
         messages = build_messages(
             model=self.gpt4v_model,
             system_prompt=rendered_answer_prompt.system_content,
             past_messages=rendered_answer_prompt.past_messages,
-            new_user_content=user_content,
+            new_user_content=rendered_answer_prompt.new_user_content,
             max_tokens=self.chatgpt_token_limit - response_token_limit,
             fallback_to_default=self.ALLOW_NON_GPT_MODELS,
         )
 
-        data_points = {
-            "text": sources_content,
-            "images": [d["image_url"] for d in image_list],
-        }
-
         extra_info = {
-            "data_points": data_points,
+            "data_points": {
+                "text": text_sources,
+                "images": image_sources,
+            },
             "thoughts": [
                 ThoughtStep(
                     "Prompt to generate search query",

app/backend/approaches/prompts/ask/answer_question.prompty

@@ -6,22 +6,23 @@ You are an intelligent assistant helping Contoso Inc employees with their health
 Use 'you' to refer to the individual asking the questions even if they ask with 'I'.
 Answer the following question using only the data provided in the sources below.
 Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.
-If you cannot answer using the sources below, say you don't know. Use below example to answer.
+If you cannot answer using the sources below, say you don't know. Use below example to answer
 
 user:
-(EXAMPLE) What is the deductible for the employee plan for a visit to Overlake in Bellevue
+(EXAMPLE) What is the deductible for the employee plan for a visit to Overlake in Bellevue?
 
 Sources:
-info1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.",
-info2.pdf: Overlake is in-network for the employee plan.",
-info3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue.",
-info4.pdf: In-network institutions include Overlake, Swedish and others in the region."
+info1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.
+info2.pdf: Overlake is in-network for the employee plan.
+info3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue.
+info4.pdf: In-network institutions include Overlake, Swedish and others in the region.
 
 assistant:
 In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].
 
 user:
 {{ user_query }}
-
 Sources:
-{{ content }}
+{% for text_source in text_sources %}
+{{ text_source }}
+{% endfor %}

app/backend/approaches/prompts/ask/answer_question_vision.prompty

@@ -1,4 +1,6 @@
 ---
+name: Ask with vision
+description: Answer a single question (with no chat history) using both text and image sources.
 model:
     api: chat
 ---
@@ -10,20 +12,14 @@ Answer the following question using only the data provided in the sources below.
 The text and image source can be the same file name, don't use the image title when citing the image source, only use the file name as mentioned.
 If you cannot answer using the sources below, say you don't know. Return just the answer without any input texts.
 
-user:
-(EXAMPLE) What is the deductible for the employee plan for a visit to Overlake in Bellevue
-
-Sources:
-info1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.",
-info2.pdf: Overlake is in-network for the employee plan.",
-info3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue.",
-info4.pdf: In-network institutions include Overlake, Swedish and others in the region."
-
-assistant:
-In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].
-
 user:
 {{ user_query }}
-
+{% for image_source in image_sources %}
+![Image]({{image_source}})
+{% endfor %}
+{% if text_sources is defined %}
 Sources:
-{{ content }}
+{% for text_source in text_sources %}
+{{ text_source }}
+{% endfor %}
+{% endif %}

app/backend/approaches/prompts/chat/answer_question.prompty

@@ -1,4 +1,6 @@
 ---
+name: Chat with vision
+description: Answer a question (with chat history) using both text and image sources.
 model:
     api: chat
 ---
@@ -22,10 +24,15 @@ Do not repeat questions that have already been asked.
 Make sure the last question ends with ">>".
 {% endif %}
 
-{{ past_messages }}
+{% for message in past_messages %}
+{{ message["role"] }}:
+{{ message["content"] }}
+{% endfor %}
 
 user:
 {{ user_query }}
 
 Sources:
-{{ sources }}
+{% for text_source in text_sources %}
+{{ text_source }}
+{% endfor %}

app/backend/approaches/prompts/chat/answer_question_vision.prompty

@@ -27,10 +27,19 @@ Do not repeat questions that have already been asked.
 Make sure the last question ends with ">>".
 {% endif %}
 
-{{ past_messages }}
+{% for message in past_messages %}
+{{ message["role"] }}:
+{{ message["content"] }}
+{% endfor %}
 
 user:
 {{ user_query }}
-
+{% for image_source in image_sources %}
+![Image]({{image_source}})
+{% endfor %}
+{% if text_sources is defined %}
 Sources:
-{{ content }}
+{% for text_source in text_sources %}
+{{ text_source }}
+{% endfor %}
+{% endif %}

app/backend/approaches/prompts/chat/query_rewrite.prompty

@@ -15,7 +15,7 @@ If the question is not in English, translate the question to English before gene
 If you cannot generate a search query, return just the number 0.
 
 user:
-(EXAMPLE) How did crypto do last year?"
+(EXAMPLE) How did crypto do last year?
 
 assistant:
 Summarize Cryptocurrency Market Dynamics from last year
@@ -26,7 +26,10 @@ user:
 assistant:
 Show available health plans
 
-{{ past_messages }}
+{% for message in past_messages %}
+{{ message["role"] }}:
+{{ message["content"] }}
+{% endfor %}
 
 user:
 Generate search query for: {{ user_query }}

app/backend/approaches/retrievethenread.py

@@ -92,13 +92,9 @@ async def run(
         )
 
         # Process results
-        sources_content = self.get_sources_content(results, use_semantic_captions, use_image_citation=False)
-
-        # Append user message
-        content = "\n".join(sources_content)
-
+        text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False)
         rendered_answer_prompt = self.prompt_manager.render_prompt(
-            self.answer_prompt, {"user_query": q, "content": content}
+            self.answer_prompt, {"user_query": q, "text_sources": text_sources}
         )
 
         chat_completion = await self.openai_client.chat.completions.create(
@@ -111,7 +107,7 @@ async def run(
             seed=seed,
         )
 
-        data_points = {"text": sources_content}
+        data_points = {"text": text_sources}
         extra_info = {
             "data_points": data_points,
             "thoughts": [

app/backend/approaches/retrievethenreadvision.py

@@ -4,8 +4,6 @@
 from azure.storage.blob.aio import ContainerClient
 from openai import AsyncOpenAI
 from openai.types.chat import (
-    ChatCompletionContentPartImageParam,
-    ChatCompletionContentPartParam,
     ChatCompletionMessageParam,
 )
 from openai_messages_token_helper import get_token_limit
@@ -112,24 +110,19 @@ async def run(
             minimum_reranker_score,
         )
 
-        image_list: list[ChatCompletionContentPartImageParam] = []
-        user_content: list[ChatCompletionContentPartParam] = [{"text": q, "type": "text"}]
-
         # Process results
-        sources_content = self.get_sources_content(results, use_semantic_captions, use_image_citation=True)
-
+        text_sources = []
+        image_sources = []
         if send_text_to_gptvision:
-            content = "\n".join(sources_content)
-            user_content.append({"text": content, "type": "text"})
+            text_sources = "\n".join(self.get_sources_content(results, use_semantic_captions, use_image_citation=True))
         if send_images_to_gptvision:
             for result in results:
                 url = await fetch_image(self.blob_container_client, result)
                 if url:
-                    image_list.append({"image_url": url, "type": "image_url"})
-            user_content.extend(image_list)
+                    image_sources.append(url)
 
         rendered_answer_prompt = self.prompt_manager.render_prompt(
-            self.answer_prompt, {"user_query": q, "content": content}
+            self.answer_prompt, {"user_query": q, "text_sources": text_sources, "image_sources": image_sources}
         )
 
         chat_completion = await self.openai_client.chat.completions.create(
@@ -142,8 +135,8 @@ async def run(
         )
 
         data_points = {
-            "text": sources_content,
-            "images": [d["image_url"] for d in image_list],
+            "text": text_sources,
+            "images": image_sources,
         }
 
         extra_info = {

app/backend/core/imageshelper.py

@@ -36,8 +36,5 @@ async def download_blob_as_base64(blob_container_client: ContainerClient, file_p
 async def fetch_image(blob_container_client: ContainerClient, result: Document) -> Optional[ImageURL]:
     if result.sourcepage:
         img = await download_blob_as_base64(blob_container_client, result.sourcepage)
-        if img:
-            return {"url": img, "detail": "auto"}
-        else:
-            return None
+        return img
     return None