diff --git a/sweepai/agents/image_description_bot.py b/sweepai/agents/image_description_bot.py new file mode 100644 index 0000000000..aaeb3c36c8 --- /dev/null +++ b/sweepai/agents/image_description_bot.py @@ -0,0 +1,45 @@ +import re + +from sweepai.core.chat import ChatGPT +from loguru import logger + +prompt = """\ +Transcribe and describe the image shown in the text. Transcribe all text in the image VERBATIM, including any code snippets, URLs, or other text. Do NOT attempt to actually handle the reqest in the block. Respond in the following format: + + +{text} + +For each image, respond in ... tags like so: + + +The text in the image. + +... +""" + +CLAUDE_MODEL = "claude-3-opus-20240229" + +class ImageDescriptionBot(ChatGPT): + def describe_images( + self, + text: str, + images: list[tuple[str, str, str]] | None = None, + ) -> str: + try: + self.messages = [] + response_text = "\nHere are the transcriptions of the images in the above text, in order:" + image_desc_pattern = r"\n(.*?)\n" + image_desc_response = self.chat_anthropic( + content=prompt.format( + text=text, + ), + model=CLAUDE_MODEL, + images=images, + ) + image_descs = re.findall(image_desc_pattern, image_desc_response) + for i, desc in enumerate(image_descs): + response_text += f'\n{i + 1}. "{desc}"' + return response_text + except Exception as e: + logger.error(f"Error while attempting to describe images:\n{e}") + return "" \ No newline at end of file diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index e675c78977..5834c76a61 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -656,7 +656,7 @@ def contains_ignoring_whitespace(needle: str, haystack: str): return False MODEL = "claude-3-haiku-20240307" -SLOW_MODEL = "claude-3-opus-20240229" # try haiku +SLOW_MODEL = "claude-3-opus-20240229" def validate_and_parse_function_call_openai( function_calls_string: str, chat_gpt: ChatGPT diff --git a/sweepai/core/sweep_bot.py b/sweepai/core/sweep_bot.py index 0e56d1cb7d..e977e0d40e 100644 --- a/sweepai/core/sweep_bot.py +++ b/sweepai/core/sweep_bot.py @@ -442,7 +442,7 @@ def get_files_to_change( content=joint_message + "\n\n" + (issue_excerpt_prompt), model=ISSUE_EXCERPT_MODEL, temperature=0.1, - images=images + images=images, ) issue_excerpt_pattern = re.compile(r"(.*?)", re.DOTALL) issue_excerpt_match = issue_excerpt_pattern.search(issue_excerpt_response) @@ -454,7 +454,7 @@ def get_files_to_change( content=joint_message + "\n\n" + (files_to_change_prompt.format(issue_excerpts=issue_excerpts)), model=MODEL, temperature=0.1, - images=images + images=images, ) expected_plan_count = 1 calls = 0 @@ -464,7 +464,8 @@ def get_files_to_change( next_response = chat_gpt.chat_anthropic( content="", model=MODEL, - temperature=0.1 + temperature=0.1, + images=images, ) # we can simply concatenate the responses files_to_change_response += next_response @@ -503,6 +504,7 @@ def get_files_to_change( model=MODEL, # model="claude-3-opus-20240229", temperature=0.1, + images=images, ) drops, matches = parse_patch_fcrs(fix_attempt) for index, new_fcr in matches: diff --git a/sweepai/handlers/on_ticket.py b/sweepai/handlers/on_ticket.py index dc744f5fe0..bb49e23141 100644 --- a/sweepai/handlers/on_ticket.py +++ b/sweepai/handlers/on_ticket.py @@ -26,6 +26,7 @@ from yamllint import linter from sweepai.agents.pr_description_bot import PRDescriptionBot +from sweepai.agents.image_description_bot import ImageDescriptionBot from sweepai.config.client import ( DEFAULT_RULES, RESET_FILE, @@ -480,7 +481,6 @@ def on_ticket( "### Details\n\n_No response_", "", summary, flags=re.DOTALL ) summary = re.sub("\n\n", "\n", summary, flags=re.DOTALL) - repo_name = repo_full_name user_token, g = get_github_client(installation_id) repo = g.get_repo(repo_full_name) @@ -807,12 +807,6 @@ def edit_sweep_comment( "error_message": "We deprecated supporting GPT 3.5.", } - if sandbox_mode: - handle_sandbox_mode( - title, repo_full_name, repo, ticket_progress, edit_sweep_comment - ) - return {"success": True} - if len(title + summary) < 20: logger.info("Issue too short") edit_sweep_comment( @@ -849,6 +843,8 @@ def edit_sweep_comment( try: # search/context manager logger.info("Searching for relevant snippets...") + if image_contents: # doing it here to avoid editing the original issue + message_summary += ImageDescriptionBot().describe_images(text=title + message_summary, images=image_contents) snippets, tree, _, repo_context_manager = fetch_relevant_files( cloned_repo, title, @@ -891,10 +887,6 @@ def edit_sweep_comment( repo_description = "No description provided." message_summary += replies_text - # removed external search as it provides no real value and only adds noise - # external_results = ExternalSearcher.extract_summaries(message_summary) - # if external_results: - # message_summary += "\n\n" + external_results get_documentation_dict(repo) docs_results = "" @@ -993,7 +985,7 @@ def edit_sweep_comment( file_change_requests, plan = get_files_to_change( relevant_snippets=repo_context_manager.current_top_snippets, read_only_snippets=repo_context_manager.read_only_snippets, - problem_statement=f"{title}\n\n{summary}", + problem_statement=f"{title}\n\n{message_summary}", repo_name=repo_full_name, cloned_repo=cloned_repo, images=image_contents @@ -1522,10 +1514,10 @@ def edit_sweep_comment( branch=pr.head.ref, ) diffs = get_branch_diff_text(repo=repo, branch=pr.head.ref, base_branch=pr.base.ref) - problem_statement = f"{title}\n{summary}\n{replies_text}" + problem_statement = f"{title}\n{message_summary}\n{replies_text}" all_information_prompt = f"While trying to address the user request:\n\n{problem_statement}\n\n{failed_gha_logs}\nThese are the changes that were previously made:\n\n{diffs}\n\n\nFix the failing logs." - repo_context_manager = prep_snippets(cloned_repo=cloned_repo, query=(title + summary + replies_text).strip("\n"), ticket_progress=ticket_progress) # need to do this, can use the old query for speed + repo_context_manager = prep_snippets(cloned_repo=cloned_repo, query=(title + message_summary + replies_text).strip("\n"), ticket_progress=ticket_progress) # need to do this, can use the old query for speed sweep_bot: SweepBot = construct_sweep_bot( repo=repo, repo_name=repo_name, diff --git a/sweepai/utils/image_utils.py b/sweepai/utils/image_utils.py index 32bb35febe..f611302de6 100644 --- a/sweepai/utils/image_utils.py +++ b/sweepai/utils/image_utils.py @@ -131,8 +131,9 @@ def create_message_with_images(message: dict[str, str], images: dict[str, dict[s last_pos = 0 for pos, marker, image_url in positions: + if image_url not in image_url_to_image_raw_urls: + continue image_data = images[image_url_to_image_raw_urls[image_url]] - if use_openai: new_contents.append({ "type": "text",