From 24350a0ed171ace670320c0583abb8848d7beff5 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 10:34:00 +0200 Subject: [PATCH 1/8] update get-action-from-context --- lavague-core/lavague/core/navigation.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lavague-core/lavague/core/navigation.py b/lavague-core/lavague/core/navigation.py index cc15f0ae..31e934f0 100644 --- a/lavague-core/lavague/core/navigation.py +++ b/lavague-core/lavague/core/navigation.py @@ -166,7 +166,14 @@ def get_action_from_context(self, context: str, query: str) -> str: """ Generate the code from a query and a context """ - prompt = self.prompt_template.format(context_str=context, query_str=query) + authorized_xpaths = extract_xpaths_from_html(context) + + prompt = self.prompt_template.format( + context_str=context, + query_str=query, + authorized_xpaths=authorized_xpaths, + ) + response = self.llm.complete(prompt).text code = self.extractor.extract(response) return code From 73cbca417c45f3ccfedfe90fb9999163aa285bf8 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 10:36:50 +0200 Subject: [PATCH 2/8] fmt --- lavague-core/lavague/core/navigation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lavague-core/lavague/core/navigation.py b/lavague-core/lavague/core/navigation.py index 31e934f0..020f2232 100644 --- a/lavague-core/lavague/core/navigation.py +++ b/lavague-core/lavague/core/navigation.py @@ -173,7 +173,7 @@ def get_action_from_context(self, context: str, query: str) -> str: query_str=query, authorized_xpaths=authorized_xpaths, ) - + response = self.llm.complete(prompt).text code = self.extractor.extract(response) return code From fe3eeb0e2142d92b2441bcfcecdd4dbda8308053 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 10:50:51 +0200 Subject: [PATCH 3/8] fix python engine --- lavague-core/lavague/core/python_engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lavague-core/lavague/core/python_engine.py b/lavague-core/lavague/core/python_engine.py index 5d2fdae2..f84eafe8 100644 --- a/lavague-core/lavague/core/python_engine.py +++ b/lavague-core/lavague/core/python_engine.py @@ -39,6 +39,7 @@ class PythonEngine(BaseEngine): ocr_llm: BaseLLM batch_size: int confidence_threshold: float + fallback_theshold: float temp_screenshots_path: str n_search_attempts: int @@ -54,6 +55,7 @@ def __init__( display: bool = False, batch_size: int = 5, confidence_threshold: float = 0.85, + fallback_threshold: float = 0.65, temp_screenshots_path="./tmp_screenshots", n_search_attemps=10, ): @@ -71,6 +73,7 @@ def __init__( self.confidence_threshold = confidence_threshold self.temp_screenshots_path = temp_screenshots_path self.n_search_attempts = n_search_attemps + self.fallback_theshold = fallback_threshold @classmethod def from_context(cls, context: Context, driver: BaseDriver): @@ -152,7 +155,7 @@ def perform_fallback(self, prompt, instruction) -> str: image_documents=screenshots, prompt=prompt ).text.strip() output_dict = self.extract_json(output) - context_score = output_dict.get("score") + context_score = output_dict.get("score", 0) output = output_dict.get("ret") memory += output @@ -208,7 +211,7 @@ def execute_instruction(self, instruction: str) -> ActionResult: try: if ( - output_dict.get("score", 0) < self.confidence_threshold + output_dict.get("score", 0) < self.fallback_theshold ): # use fallback method output = self.perform_fallback(prompt=prompt, instruction=instruction) From bf37b414271041aeb09ed28863cb05d25c763c87 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 10:53:13 +0200 Subject: [PATCH 4/8] add authorized paths to gradio --- lavague-core/lavague/core/navigation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lavague-core/lavague/core/navigation.py b/lavague-core/lavague/core/navigation.py index 020f2232..c8b52f79 100644 --- a/lavague-core/lavague/core/navigation.py +++ b/lavague-core/lavague/core/navigation.py @@ -248,8 +248,11 @@ def execute_instruction_gradio(self, instruction: str, action_engine: Any): except: pass start = time.time() + authorized_xpaths = extract_xpaths_from_html(llm_context) prompt = self.prompt_template.format( - context_str=llm_context, query_str=instruction + context_str=llm_context, + query_str=instruction, + authorized_xpaths=authorized_xpaths, ) response = self.llm.complete(prompt).text end = time.time() From 4ece722bfcde92b532598d13b527547d738333ad Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 10:58:27 +0200 Subject: [PATCH 5/8] test new extraction --- lavague-core/lavague/core/python_engine.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lavague-core/lavague/core/python_engine.py b/lavague-core/lavague/core/python_engine.py index f84eafe8..9d4a0499 100644 --- a/lavague-core/lavague/core/python_engine.py +++ b/lavague-core/lavague/core/python_engine.py @@ -21,6 +21,7 @@ from llama_index.core.base.llms.base import BaseLLM from llama_index.core.embeddings import BaseEmbedding import re +from lavague.core.extractors import JsonFromMarkdownExtractor DEFAULT_TEMPERATURE = 0.0 @@ -80,14 +81,7 @@ def from_context(cls, context: Context, driver: BaseDriver): return cls(llm=context.llm, embedding=context.embedding, driver=driver) def extract_json(self, output: str) -> Optional[dict]: - clean = ( - output.replace("'ret'", '"ret"') - .replace("'score'", '"score"') - .replace("```json", "") - .replace("```", "") - .strip() - ) - clean = re.sub(r"\n+", "\n", clean) + clean = JsonFromMarkdownExtractor.extract(output) try: output_dict = json.loads(clean) except json.JSONDecodeError as e: From 7ecd7e07a5eb91279c3cfeb06a6cc40ac7f4fcd5 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 11:20:59 +0200 Subject: [PATCH 6/8] change extraction --- lavague-core/lavague/core/python_engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lavague-core/lavague/core/python_engine.py b/lavague-core/lavague/core/python_engine.py index 9d4a0499..55d903d8 100644 --- a/lavague-core/lavague/core/python_engine.py +++ b/lavague-core/lavague/core/python_engine.py @@ -56,7 +56,7 @@ def __init__( display: bool = False, batch_size: int = 5, confidence_threshold: float = 0.85, - fallback_threshold: float = 0.65, + fallback_threshold: float = 0.85, temp_screenshots_path="./tmp_screenshots", n_search_attemps=10, ): @@ -81,7 +81,8 @@ def from_context(cls, context: Context, driver: BaseDriver): return cls(llm=context.llm, embedding=context.embedding, driver=driver) def extract_json(self, output: str) -> Optional[dict]: - clean = JsonFromMarkdownExtractor.extract(output) + extractor = JsonFromMarkdownExtractor() + clean = extractor.extract(markdown_text=output) try: output_dict = json.loads(clean) except json.JSONDecodeError as e: From f14bb1c2c93adca7621698a3cbe5043ea6709d70 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 11:29:52 +0200 Subject: [PATCH 7/8] json fix --- lavague-core/lavague/core/python_engine.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lavague-core/lavague/core/python_engine.py b/lavague-core/lavague/core/python_engine.py index 55d903d8..a99d9b69 100644 --- a/lavague-core/lavague/core/python_engine.py +++ b/lavague-core/lavague/core/python_engine.py @@ -21,7 +21,7 @@ from llama_index.core.base.llms.base import BaseLLM from llama_index.core.embeddings import BaseEmbedding import re -from lavague.core.extractors import JsonFromMarkdownExtractor +from lavague.core.extractors import DynamicExtractor DEFAULT_TEMPERATURE = 0.0 @@ -81,12 +81,12 @@ def from_context(cls, context: Context, driver: BaseDriver): return cls(llm=context.llm, embedding=context.embedding, driver=driver) def extract_json(self, output: str) -> Optional[dict]: - extractor = JsonFromMarkdownExtractor() - clean = extractor.extract(markdown_text=output) + extractor = DynamicExtractor() + clean = extractor.extract(output) try: output_dict = json.loads(clean) except json.JSONDecodeError as e: - print(f"Error extracting JSON: {e}") + print(f"Error extracting Yaml: {e}") return None return output_dict @@ -119,7 +119,7 @@ def perform_fallback(self, prompt, instruction) -> str: context_score = -1 prompt = f""" - You must respond with a dictionary in the following format: + You must respond with a JSON object in the following format: {{ "ret": "[any relevant text transcribed from the image in order to answer the query {instruction} - make sure to answer with full sentences so the reponse can be understood out of context.]", "score": [a confidence score between 0 and 1 that the necessary context has been captured in order to answer the following query] @@ -150,9 +150,10 @@ def perform_fallback(self, prompt, instruction) -> str: image_documents=screenshots, prompt=prompt ).text.strip() output_dict = self.extract_json(output) - context_score = output_dict.get("score", 0) - output = output_dict.get("ret") - memory += output + if output_dict: + context_score = output_dict.get("score", 0) + output = output_dict.get("ret") + memory += output # delete temp image folder shutil.rmtree(Path(self.temp_screenshots_path)) From 43700e693581acca2aa4cf6565c9058099c673c5 Mon Sep 17 00:00:00 2001 From: lyie28 Date: Mon, 26 Aug 2024 12:06:10 +0200 Subject: [PATCH 8/8] error message --- lavague-core/lavague/core/python_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lavague-core/lavague/core/python_engine.py b/lavague-core/lavague/core/python_engine.py index a99d9b69..9a763d51 100644 --- a/lavague-core/lavague/core/python_engine.py +++ b/lavague-core/lavague/core/python_engine.py @@ -86,7 +86,7 @@ def extract_json(self, output: str) -> Optional[dict]: try: output_dict = json.loads(clean) except json.JSONDecodeError as e: - print(f"Error extracting Yaml: {e}") + print(f"Error extracting Json: {e}") return None return output_dict