From e25801480494e89fbd44384e0eebb7cbef3f2745 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Tue, 15 Oct 2024 09:50:08 +0300 Subject: [PATCH] feat: improve prompt creator. --- src/utils/PromptCreator.ts | 68 ++++++++++++------- .../__snapshots__/PromptCreator.test.ts.snap | 67 ++++++++++-------- 2 files changed, 83 insertions(+), 52 deletions(-) diff --git a/src/utils/PromptCreator.ts b/src/utils/PromptCreator.ts index 02d7d41..5556eae 100644 --- a/src/utils/PromptCreator.ts +++ b/src/utils/PromptCreator.ts @@ -30,7 +30,6 @@ export class PromptCreator { "", "You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API.", "Please generate the minimal executable code to perform the desired intent based on the given information and context.", - "If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence.", "" ]; } @@ -46,7 +45,7 @@ export class PromptCreator { "", "### Intent to perform", "", - `Generate the minimal executable code to perform the following intent: "${intent}"`, + `Generate the minimal executable code to perform the following intent: \"${intent}\"`, "", "### View hierarchy", "", @@ -71,7 +70,7 @@ export class PromptCreator { "", ...previousSteps.map((previousStep, index) => [ `#### Step ${index + 1}`, - `- Intent: "${previousStep.step}"`, + `- Intent: \"${previousStep.step}\"`, `- Generated code:`, "```", previousStep.code, @@ -134,36 +133,55 @@ export class PromptCreator { return [ "## Instructions", "", - [ - `Generate the minimal executable code to perform the following intent: "${intent}"`, - "Use the provided API and follow the guidelines.", - "If you cannot generate the relevant code due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence.", - "Wrap the generated code with backticks, without any additional formatting." - ] - .concat(this.createVisualAssertionsInstructionIfPossible(isSnapshotImageAttached)) - .map((instruction, index) => `${index + 1}. ${instruction}`) - .join('\n'), + `Your task is to generate the minimal executable code to perform the following intent: \"${intent}\"`, "", - "### Examples of throwing an informative error:", - "```typescript", - 'throw new Error("Unable to find the \'Submit\' button in the current view hierarchy.");', - "```", + "Please follow these steps carefully:", + "", + ...this.createStepByStepInstructions(isSnapshotImageAttached).map((instruction, index) => `${index + 1}. ${instruction}`), "", + "### Examples", + "", + "#### Example of throwing an informative error:", "```typescript", - 'throw new Error("The provided intent does not contain enough information to generate the code; \'button\' is too ambiguous for matching a specific element.");', + 'throw new Error("Unable to find the \'Submit\' button element in the current context.");', "```", - "", + "" + ].concat( + isSnapshotImageAttached + ? [ + "#### Example of returning a commented visual test if the visual assertion passes:", + "```typescript", + "// Visual assertion passed based on the snapshot image.", + "```", + "" + ] + : [] + ).concat([ "Please provide your response below:" - ]; + ]); } - private createVisualAssertionsInstructionIfPossible(isSnapshotImageAttached: boolean): string[] { - if (!isSnapshotImageAttached) { - return []; + private createStepByStepInstructions(isSnapshotImageAttached: boolean): string[] { + const steps = []; + + if (isSnapshotImageAttached) { + steps.push( + "Analyze the provided intent, the view hierarchy, and the snapshot image to understand the required action.", + "Determine if the intent can be fully validated visually using the snapshot image.", + "If the intent can be visually asserted and passes the visual check, return code that only comments on the visual test.", + "If the visual assertion fails, return code that throws an informative error explaining the failure." + ); + } else { + steps.push("Analyze the provided intent and the view hierarchy to understand the required action."); } - return [ - "In case the expected behavior can be tested visually based on the provided snapshot image, there's no need to generate test code for the assertion. Instead, return code that throws an error if the visual check fails, or an empty code block if the visual check passes." - ]; + steps.push( + "If visual validation is not possible or insufficient, generate the minimal executable code required to perform the intent using the available API.", + "If you cannot generate the relevant code due to ambiguity or invalid intent, return code that throws an informative error explaining the problem in one sentence.", + "Wrap the generated code with backticks, without any additional formatting.", + "Do not provide any additional code beyond the minimal executable code required to perform the intent." + ); + + return steps; } } diff --git a/src/utils/__snapshots__/PromptCreator.test.ts.snap b/src/utils/__snapshots__/PromptCreator.test.ts.snap index cdc54d5..14a3b39 100644 --- a/src/utils/__snapshots__/PromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PromptCreator.test.ts.snap @@ -5,7 +5,6 @@ exports[`PromptCreator should create a prompt for an intent correctly 1`] = ` You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. -If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. ## Context @@ -89,19 +88,29 @@ element(by.id("uniqueId")) ## Instructions -1. Generate the minimal executable code to perform the following intent: "tap button" -2. Use the provided API and follow the guidelines. -3. If you cannot generate the relevant code due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. -4. Wrap the generated code with backticks, without any additional formatting. -5. In case the expected behavior can be tested visually based on the provided snapshot image, there's no need to generate test code for the assertion. Instead, return code that throws an error if the visual check fails, or an empty code block if the visual check passes. +Your task is to generate the minimal executable code to perform the following intent: "tap button" + +Please follow these steps carefully: + +1. Analyze the provided intent, the view hierarchy, and the snapshot image to understand the required action. +2. Determine if the intent can be fully validated visually using the snapshot image. +3. If the intent can be visually asserted and passes the visual check, return code that only comments on the visual test. +4. If the visual assertion fails, return code that throws an informative error explaining the failure. +5. If visual validation is not possible or insufficient, generate the minimal executable code required to perform the intent using the available API. +6. If you cannot generate the relevant code due to ambiguity or invalid intent, return code that throws an informative error explaining the problem in one sentence. +7. Wrap the generated code with backticks, without any additional formatting. +8. Do not provide any additional code beyond the minimal executable code required to perform the intent. -### Examples of throwing an informative error: +### Examples + +#### Example of throwing an informative error: \`\`\`typescript -throw new Error("Unable to find the 'Submit' button in the current view hierarchy."); +throw new Error("Unable to find the 'Submit' button element in the current context."); \`\`\` +#### Example of returning a commented visual test if the visual assertion passes: \`\`\`typescript -throw new Error("The provided intent does not contain enough information to generate the code; 'button' is too ambiguous for matching a specific element."); +// Visual assertion passed based on the snapshot image. \`\`\` Please provide your response below:" @@ -112,7 +121,6 @@ exports[`PromptCreator should handle when no snapshot image is attached 1`] = ` You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. -If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. ## Context @@ -192,18 +200,21 @@ element(by.id("uniqueId")) ## Instructions -1. Generate the minimal executable code to perform the following intent: "expect button to be visible" -2. Use the provided API and follow the guidelines. -3. If you cannot generate the relevant code due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. +Your task is to generate the minimal executable code to perform the following intent: "expect button to be visible" + +Please follow these steps carefully: + +1. Analyze the provided intent and the view hierarchy to understand the required action. +2. If visual validation is not possible or insufficient, generate the minimal executable code required to perform the intent using the available API. +3. If you cannot generate the relevant code due to ambiguity or invalid intent, return code that throws an informative error explaining the problem in one sentence. 4. Wrap the generated code with backticks, without any additional formatting. +5. Do not provide any additional code beyond the minimal executable code required to perform the intent. -### Examples of throwing an informative error: -\`\`\`typescript -throw new Error("Unable to find the 'Submit' button in the current view hierarchy."); -\`\`\` +### Examples +#### Example of throwing an informative error: \`\`\`typescript -throw new Error("The provided intent does not contain enough information to generate the code; 'button' is too ambiguous for matching a specific element."); +throw new Error("Unable to find the 'Submit' button element in the current context."); \`\`\` Please provide your response below:" @@ -214,7 +225,6 @@ exports[`PromptCreator should include previous intents in the context 1`] = ` You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. -If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. ## Context @@ -311,18 +321,21 @@ element(by.id("uniqueId")) ## Instructions -1. Generate the minimal executable code to perform the following intent: "tap button" -2. Use the provided API and follow the guidelines. -3. If you cannot generate the relevant code due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. +Your task is to generate the minimal executable code to perform the following intent: "tap button" + +Please follow these steps carefully: + +1. Analyze the provided intent and the view hierarchy to understand the required action. +2. If visual validation is not possible or insufficient, generate the minimal executable code required to perform the intent using the available API. +3. If you cannot generate the relevant code due to ambiguity or invalid intent, return code that throws an informative error explaining the problem in one sentence. 4. Wrap the generated code with backticks, without any additional formatting. +5. Do not provide any additional code beyond the minimal executable code required to perform the intent. -### Examples of throwing an informative error: -\`\`\`typescript -throw new Error("Unable to find the 'Submit' button in the current view hierarchy."); -\`\`\` +### Examples +#### Example of throwing an informative error: \`\`\`typescript -throw new Error("The provided intent does not contain enough information to generate the code; 'button' is too ambiguous for matching a specific element."); +throw new Error("Unable to find the 'Submit' button element in the current context."); \`\`\` Please provide your response below:"