Merge branch 'feat/planning-error-messages' of https://github.com/swe…

…epai/sweep into feat/planning-error-messages

sweepai/agents/image_description_bot.py

@@ -0,0 +1,45 @@
+import re
+
+from sweepai.core.chat import ChatGPT
+from loguru import logger
+
+prompt = """\
+Transcribe and describe the image shown in the text. Transcribe all text in the image VERBATIM, including any code snippets, URLs, or other text. Do NOT attempt to actually handle the reqest in the <text> block. Respond in the following format:
+
+<text>
+{text}
+</text>
+For each image, respond in <image_description>...</image_description> tags like so:
+<image_descriptions>
+<image_description>
+The text in the image.
+</image_description>
+...
+</image_descriptions>"""
+
+CLAUDE_MODEL = "claude-3-opus-20240229"
+
+class ImageDescriptionBot(ChatGPT):
+    def describe_images(
+        self,
+        text: str,
+        images: list[tuple[str, str, str]] | None = None,
+    ) -> str:
+        try:
+            self.messages = []
+            response_text = "\nHere are the transcriptions of the images in the above text, in order:"
+            image_desc_pattern = r"<image_description>\n(.*?)\n</image_description>"
+            image_desc_response = self.chat_anthropic(
+                content=prompt.format(
+                    text=text,
+                ),
+                model=CLAUDE_MODEL,
+                images=images,
+            )
+            image_descs = re.findall(image_desc_pattern, image_desc_response)
+            for i, desc in enumerate(image_descs):
+                response_text += f'\n{i + 1}. "{desc}"'
+            return response_text
+        except Exception as e:
+            logger.error(f"Error while attempting to describe images:\n{e}")
+            return ""

sweepai/agents/modify.py

@@ -656,7 +656,7 @@ def contains_ignoring_whitespace(needle: str, haystack: str):
     return False
 
 MODEL = "claude-3-haiku-20240307"
-SLOW_MODEL = "claude-3-opus-20240229" # try haiku
+SLOW_MODEL = "claude-3-opus-20240229"
 
 def validate_and_parse_function_call_openai(
     function_calls_string: str, chat_gpt: ChatGPT

sweepai/core/sweep_bot.py

@@ -442,7 +442,7 @@ def get_files_to_change(
             content=joint_message + "\n\n" + (issue_excerpt_prompt),
             model=ISSUE_EXCERPT_MODEL,
             temperature=0.1,
-            images=images
+            images=images,
         )
         issue_excerpt_pattern = re.compile(r"<issue_excerpts>(.*?)</issue_excerpts>", re.DOTALL)
         issue_excerpt_match = issue_excerpt_pattern.search(issue_excerpt_response)
@@ -454,7 +454,7 @@ def get_files_to_change(
             content=joint_message + "\n\n" + (files_to_change_prompt.format(issue_excerpts=issue_excerpts)),
             model=MODEL,
             temperature=0.1,
-            images=images
+            images=images,
         )
         expected_plan_count = 1
         calls = 0
@@ -464,7 +464,8 @@ def get_files_to_change(
                 next_response = chat_gpt.chat_anthropic(
                     content="",
                     model=MODEL,
-                    temperature=0.1
+                    temperature=0.1,
+                    images=images,
                 )
                 # we can simply concatenate the responses
                 files_to_change_response += next_response
@@ -503,6 +504,7 @@ def get_files_to_change(
                 model=MODEL,
                 # model="claude-3-opus-20240229",
                 temperature=0.1,
+                images=images,
             )
             drops, matches = parse_patch_fcrs(fix_attempt)
             for index, new_fcr in matches:

sweepai/handlers/on_ticket.py

@@ -26,6 +26,7 @@
 from yamllint import linter
 
 from sweepai.agents.pr_description_bot import PRDescriptionBot
+from sweepai.agents.image_description_bot import ImageDescriptionBot
 from sweepai.config.client import (
     DEFAULT_RULES,
     RESET_FILE,
@@ -480,7 +481,6 @@ def on_ticket(
             "### Details\n\n_No response_", "", summary, flags=re.DOTALL
         )
         summary = re.sub("\n\n", "\n", summary, flags=re.DOTALL)
-
         repo_name = repo_full_name
         user_token, g = get_github_client(installation_id)
         repo = g.get_repo(repo_full_name)
@@ -807,12 +807,6 @@ def edit_sweep_comment(
                     "error_message": "We deprecated supporting GPT 3.5.",
                 }
 
-            if sandbox_mode:
-                handle_sandbox_mode(
-                    title, repo_full_name, repo, ticket_progress, edit_sweep_comment
-                )
-                return {"success": True}
-
             if len(title + summary) < 20:
                 logger.info("Issue too short")
                 edit_sweep_comment(
@@ -849,6 +843,8 @@ def edit_sweep_comment(
             try:
                 # search/context manager
                 logger.info("Searching for relevant snippets...")
+                if image_contents: # doing it here to avoid editing the original issue
+                    message_summary += ImageDescriptionBot().describe_images(text=title + message_summary, images=image_contents)
                 snippets, tree, _, repo_context_manager = fetch_relevant_files(
                     cloned_repo,
                     title,
@@ -891,10 +887,6 @@ def edit_sweep_comment(
                 repo_description = "No description provided."
 
             message_summary += replies_text
-            # removed external search as it provides no real value and only adds noise
-            # external_results = ExternalSearcher.extract_summaries(message_summary)
-            # if external_results:
-            #     message_summary += "\n\n" + external_results
 
             get_documentation_dict(repo)
             docs_results = ""
@@ -993,7 +985,7 @@ def edit_sweep_comment(
                 file_change_requests, plan = get_files_to_change(
                     relevant_snippets=repo_context_manager.current_top_snippets,
                     read_only_snippets=repo_context_manager.read_only_snippets,
-                    problem_statement=f"{title}\n\n{summary}",
+                    problem_statement=f"{title}\n\n{message_summary}",
                     repo_name=repo_full_name,
                     cloned_repo=cloned_repo,
                     images=image_contents
@@ -1522,10 +1514,10 @@ def edit_sweep_comment(
                                 branch=pr.head.ref,
                             )
                             diffs = get_branch_diff_text(repo=repo, branch=pr.head.ref, base_branch=pr.base.ref)
-                            problem_statement = f"{title}\n{summary}\n{replies_text}"
+                            problem_statement = f"{title}\n{message_summary}\n{replies_text}"
                             all_information_prompt = f"While trying to address the user request:\n<user_request>\n{problem_statement}\n</user_request>\n{failed_gha_logs}\nThese are the changes that were previously made:\n<diffs>\n{diffs}\n</diffs>\n\nFix the failing logs."
 
-                            repo_context_manager = prep_snippets(cloned_repo=cloned_repo, query=(title + summary + replies_text).strip("\n"), ticket_progress=ticket_progress) # need to do this, can use the old query for speed
+                            repo_context_manager = prep_snippets(cloned_repo=cloned_repo, query=(title + message_summary + replies_text).strip("\n"), ticket_progress=ticket_progress) # need to do this, can use the old query for speed
                             sweep_bot: SweepBot = construct_sweep_bot(
                                 repo=repo,
                                 repo_name=repo_name,

sweepai/utils/image_utils.py

@@ -131,8 +131,9 @@ def create_message_with_images(message: dict[str, str], images: dict[str, dict[s
 
     last_pos = 0
     for pos, marker, image_url in positions:
+        if image_url not in image_url_to_image_raw_urls:
+            continue
         image_data = images[image_url_to_image_raw_urls[image_url]]
-
         if use_openai:
             new_contents.append({
                 "type": "text",