From ca5594f3b8058a18a3d13e4e100589a03787587d Mon Sep 17 00:00:00 2001 From: Matthias BRUNEL Date: Fri, 11 Oct 2024 00:49:18 +0200 Subject: [PATCH 1/5] agent-studio api --- .../lavague/drivers/selenium/base.py | 2 + .../lavague/exporters/python_selenium/qa.py | 24 ++++++----- lavague-sdk/lavague/sdk/action/__init__.py | 2 + lavague-sdk/lavague/sdk/action/base.py | 13 ++++++ lavague-sdk/lavague/sdk/base_driver/base.py | 1 + lavague-sdk/lavague/sdk/client.py | 27 ++++++++++++- lavague-sdk/lavague/sdk/trajectory/model.py | 40 +++++++++++++++---- 7 files changed, 90 insertions(+), 19 deletions(-) diff --git a/lavague-integrations/drivers/lavague-drivers-selenium/lavague/drivers/selenium/base.py b/lavague-integrations/drivers/lavague-drivers-selenium/lavague/drivers/selenium/base.py index a19621c0..87be0d64 100644 --- a/lavague-integrations/drivers/lavague-drivers-selenium/lavague/drivers/selenium/base.py +++ b/lavague-integrations/drivers/lavague-drivers-selenium/lavague/drivers/selenium/base.py @@ -25,6 +25,8 @@ NoPageException, ) +from lavague.sdk.action.navigation import WebNavigationAction, NavigationCommand +from lavague.sdk.action import ActionStatus from selenium.common.exceptions import ( NoSuchElementException, TimeoutException, diff --git a/lavague-integrations/exporters/lavague-exporters-python-selenium/lavague/exporters/python_selenium/qa.py b/lavague-integrations/exporters/lavague-exporters-python-selenium/lavague/exporters/python_selenium/qa.py index c1f055a7..a85d4164 100644 --- a/lavague-integrations/exporters/lavague-exporters-python-selenium/lavague/exporters/python_selenium/qa.py +++ b/lavague-integrations/exporters/lavague-exporters-python-selenium/lavague/exporters/python_selenium/qa.py @@ -92,6 +92,7 @@ Element outer HTML: {outer_html} Test:""" + class QASeleniumExporter(PythonSeleniumExporter): def __init__(self, model: str = "gpt-4o", time_between_actions: float = 2.5): self.model = model @@ -128,19 +129,23 @@ def export(self, trajectory: TrajectoryData, scenario: str) -> str: outer_html: str = output.outer_html prompt = PROMPT_TEMPLATE.format( - test_specs=scenario, + test_specs=scenario, context=context, description=description, text=text, outer_html=outer_html, ) - test_response = completion( - model = self.model, - messages = [ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": prompt} - ] - ).choices[0].message.content + test_response = ( + completion( + model=self.model, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ], + ) + .choices[0] + .message.content + ) generated_asserts: str = extract_code_block(test_response) translated_action_lines.append(generated_asserts) @@ -154,6 +159,7 @@ def export(self, trajectory: TrajectoryData, scenario: str) -> str: translated_actions_str: str = self.merge_code(*translated_actions) return self.merge_code(setup, translated_actions_str, teardown) + def extract_code_block(code_block): pattern = r"```(?:python)?\n(.*?)\n```" match = re.search(pattern, code_block, re.DOTALL) @@ -161,4 +167,4 @@ def extract_code_block(code_block): if match: return match.group(1).strip() else: - return "No code block found" \ No newline at end of file + return "No code block found" diff --git a/lavague-sdk/lavague/sdk/action/__init__.py b/lavague-sdk/lavague/sdk/action/__init__.py index 4ac94e06..6a8d3dd2 100644 --- a/lavague-sdk/lavague/sdk/action/__init__.py +++ b/lavague-sdk/lavague/sdk/action/__init__.py @@ -1,5 +1,7 @@ from lavague.sdk.action.base import ( Action, + Instruction, + EngineType, ActionType, ActionStatus, ActionParser, diff --git a/lavague-sdk/lavague/sdk/action/base.py b/lavague-sdk/lavague/sdk/action/base.py index d6a272a7..e871e712 100644 --- a/lavague-sdk/lavague/sdk/action/base.py +++ b/lavague-sdk/lavague/sdk/action/base.py @@ -14,6 +14,13 @@ class ActionType(str, Enum): EXTRACTION = "web_extraction" +class EngineType(str, Enum): + NAVIGATION = "Navigation Engine" + EXTRACTION = "Element Extraction Engine" + CONTROLS = "Navigation Controls" + COMPLETE = "COMPLETE" + + T = TypeVar("T") @@ -55,6 +62,12 @@ def parse(self, action_dict: Dict) -> Action: return Action.parse(action_dict) +class Instruction(BaseModel): + chain_of_toughts: str + engine: EngineType + engine_instruction: str + + class UnhandledTypeException(Exception): pass diff --git a/lavague-sdk/lavague/sdk/base_driver/base.py b/lavague-sdk/lavague/sdk/base_driver/base.py index 365bf4f5..2262c5af 100644 --- a/lavague-sdk/lavague/sdk/base_driver/base.py +++ b/lavague-sdk/lavague/sdk/base_driver/base.py @@ -67,6 +67,7 @@ def execute(self, action: NavigationOutput) -> None: raise NotImplementedError( f"Action {action.navigation_command} not implemented" ) + self.wait_for_idle() @abstractmethod def destroy(self) -> None: diff --git a/lavague-sdk/lavague/sdk/client.py b/lavague-sdk/lavague/sdk/client.py index b5bdf143..f765ee06 100644 --- a/lavague-sdk/lavague/sdk/client.py +++ b/lavague-sdk/lavague/sdk/client.py @@ -2,7 +2,7 @@ from typing import Any, Optional, Tuple import requests -from lavague.sdk.action import DEFAULT_PARSER, ActionParser +from lavague.sdk.action import DEFAULT_PARSER, ActionParser, Instruction, Action from lavague.sdk.trajectory import Trajectory from lavague.sdk.trajectory.controller import TrajectoryController from lavague.sdk.trajectory.model import StepCompletion @@ -86,7 +86,30 @@ def next_step(self, run_id: str) -> StepCompletion: f"/runs/{run_id}/step", "POST", ) - return StepCompletion.model_validate_json(content) + return StepCompletion.from_data(content) + + def generate_instruction(self, run_id: str) -> Instruction: + content = self.request_api( + f"/runs/{run_id}/step/instruction", + "POST", + ) + return Instruction.model_validate_json(content) + + def generate_action(self, run_id: str, instruction: Instruction) -> StepCompletion: + content = self.request_api( + f"/runs/{run_id}/step/action", + "POST", + instruction.model_dump(), + ) + return StepCompletion.from_data(content) + + def execute_action(self, run_id: str, action: StepCompletion) -> StepCompletion: + content = self.request_api( + f"/runs/{run_id}/step/execution", + "POST", + action.model_dump(), + ) + return StepCompletion.from_data(content) def stop(self, run_id: str) -> None: self.request_api( diff --git a/lavague-sdk/lavague/sdk/trajectory/model.py b/lavague-sdk/lavague/sdk/trajectory/model.py index 4e0ebe3c..caa64af5 100644 --- a/lavague-sdk/lavague/sdk/trajectory/model.py +++ b/lavague-sdk/lavague/sdk/trajectory/model.py @@ -1,9 +1,11 @@ from enum import Enum from typing import Any, Dict, List, Tuple, Optional from pydantic import BaseModel, SerializeAsAny -from lavague.sdk.action import Action +from lavague.sdk.action import Action, ActionParser from lavague.sdk.action.base import DEFAULT_PARSER from pydantic import model_validator +from pydantic_core import from_json + class RunStatus(str, Enum): STARTING = "starting" @@ -31,24 +33,26 @@ class TrajectoryData(BaseModel): actions: List[SerializeAsAny[Action]] error_msg: Optional[str] = None - @model_validator(mode='before') + @model_validator(mode="before") @classmethod def deserialize_actions(cls, values: Dict[str, Any]) -> Dict[str, Any]: - if 'actions' in values: - actions = values['actions'] + if "actions" in values: + actions = values["actions"] deserialized_actions = [] for action_data in actions: if isinstance(action_data, Action): deserialized_actions.append(action_data) continue - action_type = action_data.get('action_type') + action_type = action_data.get("action_type") if action_type: - action_class = DEFAULT_PARSER.engine_action_builders.get(action_type, Action) + action_class = DEFAULT_PARSER.engine_action_builders.get( + action_type, Action + ) deserialized_action = action_class.parse(action_data) deserialized_actions.append(deserialized_action) else: deserialized_actions.append(Action.parse(action_data)) - values['actions'] = deserialized_actions + values["actions"] = deserialized_actions return values def write_to_file(self, file_path: str): @@ -56,7 +60,27 @@ def write_to_file(self, file_path: str): with open(file_path, "w", encoding="utf-8") as file: file.write(json_model) + class StepCompletion(BaseModel): run_status: RunStatus action: Optional[Action] - run_mode: RunMode \ No newline at end of file + run_mode: RunMode + + @classmethod + def from_data( + cls, + data: str | bytes | bytearray, + parser: ActionParser = DEFAULT_PARSER, + ): + obj = from_json(data) + return cls.from_dict(obj, parser) + + @classmethod + def from_dict( + cls, + data: Dict, + parser: ActionParser = DEFAULT_PARSER, + ): + action = data.get("action") + action = parser.parse(action) if action else None + return cls.model_validate({**data, "action": action}) From a8e0c5728a441281ed7135c0820c1d1678406c5b Mon Sep 17 00:00:00 2001 From: Alexis Deprez Date: Fri, 11 Oct 2024 18:38:39 +0200 Subject: [PATCH 2/5] feat: add knowledge --- lavague-sdk/lavague/sdk/client.py | 6 ++--- lavague-sdk/lavague/sdk/trajectory/model.py | 28 ++++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lavague-sdk/lavague/sdk/client.py b/lavague-sdk/lavague/sdk/client.py index f765ee06..0a8d34ac 100644 --- a/lavague-sdk/lavague/sdk/client.py +++ b/lavague-sdk/lavague/sdk/client.py @@ -2,10 +2,10 @@ from typing import Any, Optional, Tuple import requests -from lavague.sdk.action import DEFAULT_PARSER, ActionParser, Instruction, Action +from lavague.sdk.action import DEFAULT_PARSER, ActionParser, Instruction from lavague.sdk.trajectory import Trajectory from lavague.sdk.trajectory.controller import TrajectoryController -from lavague.sdk.trajectory.model import StepCompletion +from lavague.sdk.trajectory.model import StepCompletion, StepKnowledge from lavague.sdk.utilities.config import LAVAGUE_API_BASE_URL, get_config, is_flag_true from PIL import Image, ImageFile from pydantic import BaseModel @@ -103,7 +103,7 @@ def generate_action(self, run_id: str, instruction: Instruction) -> StepCompleti ) return StepCompletion.from_data(content) - def execute_action(self, run_id: str, action: StepCompletion) -> StepCompletion: + def execute_action(self, run_id: str, action: StepKnowledge) -> StepCompletion: content = self.request_api( f"/runs/{run_id}/step/execution", "POST", diff --git a/lavague-sdk/lavague/sdk/trajectory/model.py b/lavague-sdk/lavague/sdk/trajectory/model.py index caa64af5..35cd39ef 100644 --- a/lavague-sdk/lavague/sdk/trajectory/model.py +++ b/lavague-sdk/lavague/sdk/trajectory/model.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, Dict, List, Tuple, Optional from pydantic import BaseModel, SerializeAsAny -from lavague.sdk.action import Action, ActionParser +from lavague.sdk.action import Action, ActionParser, Instruction from lavague.sdk.action.base import DEFAULT_PARSER from pydantic import model_validator from pydantic_core import from_json @@ -61,10 +61,8 @@ def write_to_file(self, file_path: str): file.write(json_model) -class StepCompletion(BaseModel): - run_status: RunStatus +class ActionWrapper(BaseModel): action: Optional[Action] - run_mode: RunMode @classmethod def from_data( @@ -84,3 +82,25 @@ def from_dict( action = data.get("action") action = parser.parse(action) if action else None return cls.model_validate({**data, "action": action}) + + @model_validator(mode="before") + @classmethod + def deserialize_action(cls, values: Dict[str, Any]) -> Dict[str, Any]: + if "action" in values: + action_data = values["action"] + if not isinstance(action_data, Action) and "action_type" in action_data: + action_class = DEFAULT_PARSER.engine_action_builders.get( + action_data["action_type"], Action + ) + deserialized_action = action_class.parse(action_data) + values["action"] = deserialized_action + return values + + +class StepCompletion(ActionWrapper): + run_status: RunStatus + run_mode: RunMode + + +class StepKnowledge(ActionWrapper): + instruction: Instruction From ddaabc7dbcfb77e0500d5370e36e870277fc4a71 Mon Sep 17 00:00:00 2001 From: Alexis Deprez Date: Mon, 14 Oct 2024 11:20:18 +0200 Subject: [PATCH 3/5] fix: allow None action in ActionWrapper --- lavague-sdk/lavague/sdk/trajectory/model.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lavague-sdk/lavague/sdk/trajectory/model.py b/lavague-sdk/lavague/sdk/trajectory/model.py index 35cd39ef..d8157ff6 100644 --- a/lavague-sdk/lavague/sdk/trajectory/model.py +++ b/lavague-sdk/lavague/sdk/trajectory/model.py @@ -88,7 +88,11 @@ def from_dict( def deserialize_action(cls, values: Dict[str, Any]) -> Dict[str, Any]: if "action" in values: action_data = values["action"] - if not isinstance(action_data, Action) and "action_type" in action_data: + if ( + action_data + and not isinstance(action_data, Action) + and "action_type" in action_data + ): action_class = DEFAULT_PARSER.engine_action_builders.get( action_data["action_type"], Action ) From 2c4398005709c6de4f5cc111a988714c009fb6f2 Mon Sep 17 00:00:00 2001 From: Alexis Deprez Date: Mon, 14 Oct 2024 11:48:38 +0200 Subject: [PATCH 4/5] feat: add export step to knowledge --- lavague-sdk/lavague/sdk/trajectory/model.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lavague-sdk/lavague/sdk/trajectory/model.py b/lavague-sdk/lavague/sdk/trajectory/model.py index d8157ff6..9e955fcd 100644 --- a/lavague-sdk/lavague/sdk/trajectory/model.py +++ b/lavague-sdk/lavague/sdk/trajectory/model.py @@ -101,10 +101,13 @@ def deserialize_action(cls, values: Dict[str, Any]) -> Dict[str, Any]: return values +class StepKnowledge(ActionWrapper): + instruction: Instruction + + class StepCompletion(ActionWrapper): run_status: RunStatus run_mode: RunMode - -class StepKnowledge(ActionWrapper): - instruction: Instruction + def to_knowledge(self, instruction: Instruction) -> StepKnowledge: + return StepKnowledge(instruction=instruction, action=self.action) \ No newline at end of file From 32fdad6078a063c8b835678902c5df6ff1ea5bee Mon Sep 17 00:00:00 2001 From: Alexis Deprez Date: Mon, 14 Oct 2024 11:49:40 +0200 Subject: [PATCH 5/5] chore: format --- lavague-sdk/lavague/sdk/trajectory/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lavague-sdk/lavague/sdk/trajectory/model.py b/lavague-sdk/lavague/sdk/trajectory/model.py index 9e955fcd..d7894019 100644 --- a/lavague-sdk/lavague/sdk/trajectory/model.py +++ b/lavague-sdk/lavague/sdk/trajectory/model.py @@ -110,4 +110,4 @@ class StepCompletion(ActionWrapper): run_mode: RunMode def to_knowledge(self, instruction: Instruction) -> StepKnowledge: - return StepKnowledge(instruction=instruction, action=self.action) \ No newline at end of file + return StepKnowledge(instruction=instruction, action=self.action)