diff --git a/pyproject.toml b/pyproject.toml index ef3c580f53..233a8ec722 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "backoff==2.2.1", "pymongo==4.6.3", "gitpython==3.1.42", + "stringzilla==3.8.4", "tree-sitter==0.21.0", "tree-sitter-python==0.21.0", "tree-sitter-javascript==0.21.0", diff --git a/requirements.txt b/requirements.txt index 5886626a2b..4c164150f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -552,6 +552,7 @@ starlette==0.36.3 # via # fastapi # prometheus-fastapi-instrumentator +stringzilla==3.8.4 tabulate==0.9.0 tenacity==8.2.3 # via voyageai diff --git a/sweepai/agents/modify.py b/sweepai/agents/modify.py index d3e7aeeaa6..a9008032dd 100644 --- a/sweepai/agents/modify.py +++ b/sweepai/agents/modify.py @@ -2,8 +2,10 @@ from math import inf import os import re +import sys from rapidfuzz import fuzz, process +import stringzilla as sz from loguru import logger import rapidfuzz @@ -444,7 +446,7 @@ def test_multiply_negative_by_negative(self): Then, call the make_change function to fix the linter warnings. If the warning cannot be resolved, call submit_task with an explanation of the issue.""" -fix_syntax_prompt = """You must resolve the issue by following these steps: +fix_syntax_prompt = """You MUST resolve the issue by following these steps: # 1. Thinking @@ -455,7 +457,7 @@ def test_multiply_negative_by_negative(self): # 2. Function call -Reinvoke the make_change function call with different changes that yields valid code.""" +Then, reinvoke the make_change function call with different changes that yields valid code.""" ORIGINAL_CODE_NOT_FOUND_PROMPT = """The original_code provided does not appear to be present in file {file_path}. Your provided original_code erroneously contains: ``` @@ -487,15 +489,22 @@ def test_multiply_negative_by_negative(self): # Function call Then, follow up with a make_change function call with the corrected parameters. If you are unable to find the correct section of code, call the submit_task function with an explanation of the issue.""" -MULTIPLE_OCCURRENCES_PROMPT = """Resolve this error by following these steps: +MULTIPLE_OCCURRENCES_PROMPT = """You MUST resolve this error by following these steps: # 1. Thinking a. Identify whether you want to replace all occurrences of the original code or only a specific one. If you want to replace all occurrences, you can use the replace_all flag by adding true to the function arguments. b. If you want to replace only a specific occurrence, which occurrence you want to replace and the corresponding surrounding context, following this format: + +Corrected original code: ``` The original_code block you want to replace with surrounding context. ``` + +Corrected new code: +``` +The new_code block you want to replace with the same surrounding context. +``` # 2. Function Call @@ -552,59 +561,20 @@ def rstrip_lines(text: str) -> str: return "\n".join([line.rstrip() for line in text.split("\n")]) def indent(text: str, spaces: int) -> str: - return "\n".join([f"{' ' * spaces}{line}" for line in text.split("\n")]) + return "\n".join([f"{' ' * spaces}{line}" if line.strip() else "" for line in text.split("\n")]) def tokenize_code(code: str): - # Split on all non-alphanumeric characters - tokens = [] - current_token = "" - available_set = ("(", ")", "{", "}", "[", "]", "_") - for char in code: - if char.isalnum() or char in available_set: - current_token += char - elif current_token: - tokens.append(current_token) - current_token = "" - if current_token: - tokens.append(current_token) - return tokens + cleaned_code = "" + for line in code.split("\n"): + stripped_line = line.strip() + if stripped_line.startswith("#") or stripped_line.startswith("//") or len(stripped_line) == 0: + continue + cleaned_code += line + "\n" + return [str(token) for token in sz.Str(cleaned_code).split_charset(separator=' \n\t\r()\{\}\[\]', maxsplit=sys.maxsize, keepseparator=True) if str(token).strip()] def code_processor(code: str): return " ".join(tokenize_code(code)) -def find_best_match(needle: str, haystack: str, threshold: int = 60, verbose=True, tokenized=False): - best_match = 0 - best_score = 0 - file_contents_lines = haystack.split("\n") - num_lines = len(file_contents_lines) - num_match_lines = len(needle.split("\n")) - for start_line in tqdm(range(num_lines), total=num_lines) if verbose else range(num_lines): - potential_choices = [] - for end_line in range(start_line + max(1, num_match_lines - 5), start_line + num_match_lines + 5): - if end_line > num_lines: - break - potential_choice = "\n".join(file_contents_lines[start_line:end_line]) - potential_choices.append(potential_choice) - - results = process.extractOne( - needle, - potential_choices, - scorer=fuzz.QRatio, - score_cutoff=threshold, - processor=code_processor if tokenized else None, - ) - - if results is not None: - choice, score, _index = results - - if score > best_score: - best_score = score - best_match = choice - - if best_score > threshold: - return best_match, best_score - return "", 0 - def find_best_matches( needle: str, haystack: str, @@ -640,7 +610,7 @@ def find_best_matches( scorer=fuzz.QRatio, score_cutoff=threshold, limit=num_matches, - processor=code_processor if tokenized else None, + processor=tokenize_code if tokenized else None, **kwargs ) @@ -658,6 +628,12 @@ def find_best_matches( deduped_best_matches.append((match, score)) return deduped_best_matches[:num_matches] +def find_best_match(*args, **kwargs): + results = find_best_matches(*args, **kwargs) + if len(results) > 0: + return results[0] + return "", 0 + def find_max_indentation(needle: str): max_indent = 0 for line in needle.splitlines(): @@ -673,7 +649,10 @@ def contains_ignoring_whitespace(needle: str, haystack: str): for indent_size in range(0, max_indent + 2, 2): indented_needle = indent(needle, indent_size) if indented_needle in haystack: - return True + start_char = haystack.index(indented_needle) + start_line = haystack[:start_char].count("\n") + end_line = start_line + indented_needle.count("\n") + 1 + return start_line, end_line return False MODEL = "claude-3-haiku-20240307" @@ -869,11 +848,11 @@ def get_replaces_per_fcr(fcr: FileChangeRequest) -> int: def parse_fcr(fcr: FileChangeRequest): flags = "" justification, *_ = fcr.instructions.split("", 1) - original_code_pattern = r"(.*?)" - new_code_pattern = r"(.*?)" + original_code_pattern = r"\s*\n(.*?)" + new_code_pattern = r"\s*\n(.*?)" original_code_matches = list(re.finditer(original_code_pattern, fcr.instructions, re.DOTALL)) new_code_matches = list(re.finditer(new_code_pattern, fcr.instructions, re.DOTALL)) - replace_all_pattern = r"(.*?)" + replace_all_pattern = r"true" replace_all_matches = list(re.finditer(replace_all_pattern, fcr.instructions, re.DOTALL)) if replace_all_matches: flags += "\ntrue" @@ -1107,7 +1086,7 @@ def modify( )) # if previous things go wrong we make llm call if not function_calls_string: - model = MODEL if llm_state["attempt_count"] < 5 else SLOW_MODEL + model = MODEL if llm_state["attempt_count"] < 3 else SLOW_MODEL logger.info(f"Using model: {model}") function_calls_string = chat_gpt.chat_anthropic( content=function_output, @@ -1116,9 +1095,9 @@ def modify( use_openai=use_openai, ) if function_calls_string in llm_state["visited_set"]: - if llm_state["attempt_count"] < 5: + if llm_state["attempt_count"] < 3: logger.warning(f"Function call {function_calls_string} has already been visited, retrying with a different model.") - llm_state["attempt_count"] = 5 + llm_state["attempt_count"] = 3 function_calls_string = chat_gpt.chat_anthropic( content=SLOW_MODEL, model=model, @@ -1316,8 +1295,6 @@ def handle_function_call( second_diff_text = surrounding_lines_before + START_MARKER + best_match + END_MARKER + surrounding_lines_after best_match_diff = generate_diff(first_diff_text, second_diff_text, n=20) # this is bounded to 14 * 2 lines of context error_message = f"The original_code provided does not appear to be present in file {file_name}. Your provided original_code contains:\n```\n{tool_call['original_code']}\n```\nDid you mean the following?\n```\n{best_match}\n```\nHere is the difference between the original_code and the most similar existing code from the file, along with its surrounding code:\n```\n{best_match_diff}\n```\n" + DID_YOU_MEAN_PROMPT - # error_message = f"The original_code provided does not appear to be present in file {file_name}. Your provided original_code contains:\n```\n{tool_call['original_code']}\n```\nHere is the diff and surrounding code:\n```\n{best_match_diff}\n```" - manual_code_check(file_contents, original_code) else: # check other files, this code should skip if there are no other files all_file_contents = list(dict.fromkeys([get_latest_contents(fcr.filename, cloned_repo, modify_files_dict) for fcr in llm_state["fcrs"] if fcr.filename != file_name])) @@ -1354,7 +1331,10 @@ def handle_function_call( else: original_code_lines = original_code.split("\n") if len(original_code_lines) > 1: - original_code = "\n".join(f'{correct_indent * " "}{line}' for line in original_code_lines) + """This will match the whitespace from the code file itself""" + best_span = contains_ignoring_whitespace(original_code, file_contents) + start_line, end_line = best_span + original_code = "\n".join(file_contents.split("\n")[start_line:end_line]) else: original_code = f'{correct_indent * " "}{original_code.lstrip()}' # before we apply changes make sure original_code is unique inside current_chunk @@ -1405,7 +1385,7 @@ def handle_function_call( if original_code not in file_contents: new_correct_indent, new_rstrip_original_code = manual_code_check(file_contents, new_code) if new_correct_indent == -1: - error_message = f"The original_code provided does not appear to be present in file {file_name}. Your provided original_code contains:\n```\n{tool_call['original_code']}\n```\nBut this section of code was not found anywhere inside the current file. DOUBLE CHECK that the change you are trying to make is not already implemented in the code!" + error_message = f"The original_code provided does not appear to be present in file {file_name}. Your provided original_code contains:\n```\n{tool_call['original_code']}\n```\nBut this section of code was not found anywhere inside the current file." else: error_message = f"The original_code provided does not appear to be present in file {file_name}. However, the new_code provided is present in the file. If you would like to apply this change, please provide the correct original_code. Otherwise, call submit_task to move on to the next task." break @@ -1435,6 +1415,8 @@ def handle_function_call( ) if failing_parse: error_message = f"Error: Invalid code changes have been applied. You requested the following changes:\n\n```diff\n{current_diff}\n```\n\nBut it produces invalid code with the following error logs:\n```\n{failing_parse}\n```\n\n" + fix_syntax_prompt + # print(error_message) + # breakpoint() break elif check_results_message: warning_message = check_results_message @@ -1474,6 +1456,7 @@ def handle_function_call( # breakpoint() modify_files_dict[file_name]['contents'] = new_file_contents llm_state["attempt_lazy_change"] = False # no longer attempt lazy change + # breakpoint() elif llm_state["completed_changes_per_fcr"][current_fcr_index] + 1 < llm_state["changes_per_fcr"][current_fcr_index]: # Incomplete changes, should use a different prompt realistically llm_response = f"SUCCESS\n\nThe following changes have been applied:\n\n```diff\n{generate_diff(file_contents, new_file_contents, n=25)}\n```\n{self_review_prompt.format(current_task=llm_state['current_task'])}" diff --git a/sweepai/core/prompts.py b/sweepai/core/prompts.py index 75095b120b..536eb45004 100644 --- a/sweepai/core/prompts.py +++ b/sweepai/core/prompts.py @@ -283,19 +283,20 @@ For each error, identify what went wrong and what the fix is. Analyze the contents of the provided file path to find the correct code block that needs to be modified. Update the block with the actual code from the file, and then provide the necessary changes in the block. Follow the format: -Error #1: Description of the error +Error #0: Description of the error You will first think step-by-step about the error, and then either rewrite the instructions with the corrected fix, or drop the task. -Analyze what went wrong, including the file path and the specific code block that needs to be modified. The fix for error #1. +Analyze extremely carefully in great detail what went wrong, including the file path and the specific code block that needs to be modified. If you have failed to copy code verbatim, indicate precisely what is different between the code you provided and the code in the actual file. + + +Then, let's resolve the errors in your proposed plan: + +If you would like patch the corresponding task of the plan, create a modify block with an index. The index should be equivalent to the error number of this error_resolution block. -Strategies: -- For " does not exist in the file" errors: - - Look closely and carefully at the actual contents of the file. Are there any missing indentation, whitespace or comments? - - Look closely at similar files, could you have selected the wrong file? -- For empty blocks: - - First, look closely at the file to determine where to make the change. Then, copy that code into the . Then, copy the code into with the code you would like to add before or after the code in the block to prepend or append code. + +Rewritten instructions to resolve the error. Update the original_code and new_code blocks as required, ensuring that the block contains the actual code from the file. Update with the necessary changes: @@ -306,19 +307,11 @@ Updated new code, based on the corrections in . Ensure all newly introduced indents and comments are propagated here. - - -Let's resolve the errors in your proposed plan: + -If you determine that this task is not needed, you may drop the task like so: +Otherwise, if you absolutely cannot resolve the error, drop the task like so: Index of the task to drop - -Otherwise, you must patch the task to resolve the error like so: - - -Rewritten instructions to resolve the error. Update the original_code and new_code blocks as required, ensuring that the block contains the actual code from the file. - [additional blocks as needed, for the same file or different files] @@ -443,6 +436,7 @@ Step 1. Reference the original code in tags, copying them VERBATIM from the file, with correct indentation and whitespace. - Do NOT paraphrase or abbreviate the source code. - Placeholder comments like "# existing code" are not permitted. + - Start with a function header. Step 2. Write the new code in tags, specifying necessary imports and including relevant type definitions, interfaces, and schemas. - BE EXACT as this code will replace the mentioned . Step 3. Determine if this is a change that occurs EXACTLY in other parts of the same file. If so, add a true flag. @@ -497,12 +491,12 @@ Then, based on the analysis, propose a fix by following the format below. If the error has already been fixed, you can skip this step. - + Instructions for modifying one section of the file. Each block must have exactly one original_code and one new_code block. Do not make a change that has already been made by the intern. a. Describe the section of code that needs to be modified, i.e. the test case that checks if `foo` == `bar`. -Copy the original_code here VERBATIM from the file. Do NOT paraphrase or abbreviate the source code. Placeholder comments like "# existing code" are not permitted. +Copy the original_code here VERBATIM from the file. Do NOT paraphrase or abbreviate the source code. Placeholder comments like "# existing code" are not permitted. Start with a function header. b. Describe the changes that need to be made to the code, i.e. the test case should instead check if `foo` != `baz`. diff --git a/sweepai/core/sweep_bot.py b/sweepai/core/sweep_bot.py index bc4fa33381..e977e0d40e 100644 --- a/sweepai/core/sweep_bot.py +++ b/sweepai/core/sweep_bot.py @@ -180,9 +180,6 @@ def get_error_message( file_change_requests: list[FileChangeRequest], cloned_repo: ClonedRepo, ): - """ - TODO: - """ error_message = "" error_indices = [] for i, file_change_request in enumerate(file_change_requests): @@ -192,11 +189,11 @@ def get_error_message( parsed_fcr = parse_fcr(file_change_request) if not parsed_fcr["original_code"]: # breakpoint() - error_message += f"\nYou forgot to provide both an block. If you would like to drop this task use the marker.\n\n\n" + error_message += f"\nYou forgot to provide both an block. Here is what you provided in the instructions:\n```\n{file_change_request.instructions}\n```\nIf you would like to drop this task use the marker.\n\n\n" error_indices.append(i) continue if not parsed_fcr["new_code"]: - error_message += f"\nYou forgot to a block. If you would like to drop this task use the marker.\n\n\n" + error_message += f"\nYou forgot to a block. Here is what you provided in the instructions:\n```\n{file_change_request.instructions}\n```\nIf you would like to drop this task use the marker.\n\n\n" error_indices.append(i) continue original_code = parsed_fcr["original_code"][0].strip("\n") @@ -220,15 +217,16 @@ def get_error_message( continue if best_score > threshold: if best_score > 80: - error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify the following code instead?\n```\n{best_match}\n```\nHere is the diff between your proposed and the most similar code in the file:\n```diff\n{generate_diff(indent(original_code, best_indent), best_match)}\n```\n\n\n" + error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify the following code instead?\n```\n{best_match}\n```\nHere is the diff between your proposed and the most similar code in the file:\n```diff\n{generate_diff(indent(original_code, best_indent), best_match, n=10)}\n```\n\n\n" elif best_score > 50: - best_matches = find_best_matches(original_code, file_contents, threshold=60, tokenized=True) + best_matches = find_best_matches(original_code, file_contents, threshold=threshold, tokenized=True) + ellipses_message = "You must copy code out in full and may not use ellipses, abbreviations, or any short-hand notation in your code." if "... " in original_code else "" if len(best_matches) > 1: best_matches_string = "\n\n".join([f"Code match {i}:\n```\n{match_}\n```" for i, (match_, score) in enumerate(best_matches)]) - error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify one of the following pieces of code instead?\n{best_matches_string}\n\n\n" + error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify one of the following pieces of code instead?\n{best_matches_string}\n{ellipses_message}\n\n" else: # Same as case > 80 - error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify the following code instead?\n```\n{best_match}\n```\nHere is the diff between your proposed and the most similar code in the file:\n```diff\n{generate_diff(indent(original_code, best_indent), best_match)}\n```\n\n\n" + error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify the following code instead?\n```\n{best_match}\n```\nHere is the diff between your proposed and the most similar code in the file:\n```diff\n{generate_diff(indent(original_code, best_indent), best_match, n=10)}\n```\n{ellipses_message}\n\n" else: error_message += f"\n does not exist in `{file_change_request.filename}`. Your proposed contains:\n```\n{indent(original_code, best_indent)}\n```\nDid you mean to modify the following code instead?\n```\n{best_match}\n```\n\n\n" else: @@ -498,7 +496,9 @@ def get_files_to_change( error_message, error_indices = get_error_message(file_change_requests, cloned_repo) - if error_message: + for _ in range(3): + if not error_message: + break fix_attempt = chat_gpt.chat_anthropic( content=fix_files_to_change_prompt.format(error_message=error_message), model=MODEL, @@ -517,9 +517,9 @@ def get_files_to_change( logger.warning(f"Index {drop} not in error indices") continue file_change_requests.pop(error_indices[drop]) - new_error_message, new_error_indices = get_error_message(file_change_requests, cloned_repo) logger.debug("Old indices", error_indices) - logger.debug("New indices", new_error_indices) + error_message, error_indices = get_error_message(file_change_requests, cloned_repo) + logger.debug("New indices", error_indices) validate_file_change_requests(file_change_requests, cloned_repo) return file_change_requests, files_to_change_response @@ -1028,7 +1028,6 @@ def get_files_to_change_for_gha( ], ) MODEL = "claude-3-opus-20240229" if not use_faster_model else "claude-3-sonnet-20240229" - # MODEL = "claude-3-opus-20240229" if not use_faster_model else "claude-3-haiku-20240307" files_to_change_response = chat_gpt.chat_anthropic( content=joint_message + "\n\n" + gha_files_to_change_prompt, model=MODEL, @@ -1072,10 +1071,11 @@ def get_files_to_change_for_gha( file_change_request.raw_relevant_files = " ".join(relevant_modules) file_change_requests.append(file_change_request) - # Auto-fix plan, should do this multiple times but once works for now. error_message, error_indices = get_error_message(file_change_requests, cloned_repo) - if error_message: + for _ in range(3): + if not error_message: + break fix_attempt = chat_gpt.chat_anthropic( content=fix_files_to_change_prompt.format(error_message=error_message), model=MODEL, @@ -1093,9 +1093,10 @@ def get_files_to_change_for_gha( logger.warning(f"Index {drop} not in error indices") continue file_change_requests.pop(error_indices[drop]) - new_error_message, new_error_indices = get_error_message(file_change_requests, cloned_repo) logger.debug("Old indices", error_indices) - logger.debug("New indices", new_error_indices) + error_message, error_indices = get_error_message(file_change_requests, cloned_repo) + logger.debug("New indices", error_indices) + breakpoint() validate_file_change_requests(file_change_requests, cloned_repo) return file_change_requests, files_to_change_response diff --git a/sweepai/utils/modify_utils.py b/sweepai/utils/modify_utils.py index b01b85a978..1a2da76c56 100644 --- a/sweepai/utils/modify_utils.py +++ b/sweepai/utils/modify_utils.py @@ -85,14 +85,14 @@ def manual_code_check(file_contents: str, code_snippet: str) -> tuple[int, bool] # assume one indent is two spaces and check max 10 indents for indent in range(0, 40, 2): - new_code_lines = [f"{' ' * indent}{line}" for line in code_lines] + new_code_lines = [f"{' ' * indent}{line}" if line.strip() else "" for line in code_lines] new_code = "\n".join(new_code_lines) if new_code in file_contents: return indent, False # sometimes llm returns code with trailing whitespace, if we have reached here check again but strip all trailing whitespace code_lines = [line.rstrip() for line in code_snippet.split("\n")] for indent in range(0, 40, 2): - new_code_lines = [f"{' ' * indent}{line}" for line in code_lines] + new_code_lines = [f"{' ' * indent}{line}" if line.strip() else "" for line in code_lines] new_code = "\n".join(new_code_lines) if new_code in file_contents: return indent, True diff --git a/sweepai/utils/utils.py b/sweepai/utils/utils.py index 93b0e5ab39..4a7cfc5405 100644 --- a/sweepai/utils/utils.py +++ b/sweepai/utils/utils.py @@ -222,7 +222,7 @@ def get_new_lint_errors_for_pylint(new_errors: str, old_errors: str) -> str: results = [] for line in additional_errors: _file_delimiter, error_type, *_ = line.split(" ") - if not error_type.startswith("E") and old_error_types.count(error_type) < 2: # if there are more than 1 of the same error, we consider it new + if error_type.startswith("E") or old_error_types.count(error_type) < 2: # if there are more than 1 of the same error, we consider it new results.append(line) return "\n".join(results) @@ -414,6 +414,7 @@ def find_deepest_error(node: Node) -> Optional[Node]: @file_cache() def get_pylint_check_results(file_path: str, code: str) -> CheckResults: + logger.debug(f"Running pylint on {file_path}...") file_hash = uuid.uuid4().hex new_file = os.path.join("/tmp", file_hash + "_" + os.path.basename(file_path)) stem = os.path.splitext(os.path.basename(file_path))[0] @@ -429,7 +430,6 @@ def get_pylint_check_results(file_path: str, code: str) -> CheckResults: "--disable=R", "--disable=import-error", "--disable=no-member", - "--disable=unused-import" # we have a workaround for this tbh ], reporter=reporter, exit=False, @@ -444,18 +444,19 @@ def get_pylint_check_results(file_path: str, code: str) -> CheckResults: error_message = error_message.replace(new_file, file_path).replace(f"{file_hash}_" + stem, stem) error_message = error_message.split("-----------------------------------", 1)[0].strip() error_message = f"> pylint {file_path}\n\n" + error_message + logger.debug("Done running pylint.") return CheckResults(pylint=error_message if not succeeded else "") -@file_cache() def get_check_results(file_path: str, code: str) -> CheckResults: is_valid, error_message = check_syntax(file_path, code) if not is_valid: return CheckResults(parse_error_message=error_message) - ext = file_path.split(".")[-1] # noqa + ext = file_path.rsplit(".")[-1] # noqa if ext == "py": try: return get_pylint_check_results(file_path, code) except Exception as e: + breakpoint() logger.exception(e) elif ext in ["js", "jsx", "ts", "tsx"]: # see if eslint is installed