Reproducibility

README.md

@@ -54,6 +54,33 @@ export MINIWOB_URL="file://$HOME/dev/miniwob-plusplus/miniwob/html/miniwob/"
 ```
 </details>
 
+<details>
+
+<summary>WorkArena</summary>
+
+See [detailed instructions on workarena github](https://github.com/ServiceNow/WorkArena?tab=readme-ov-file#getting-started)
+
+At a glance: 
+* [Sign in](https://developer.servicenow.com/) and reqeuest a `washington` instance.
+* Once the instance is ready, you should see `<your instance URL>` and `<your-instance-password>`
+* Add these to your `.bashrc` (or `.zshrc`) and `source` it (note: make sure that
+  all variables are in single quotes unless you happen to have a password with a
+  single quote in it)
+```bash
+export SNOW_INSTANCE_URL='https://<your-instance-number>.service-now.com/'
+export SNOW_INSTANCE_UNAME='admin'
+export SNOW_INSTANCE_PWD='<your-instance-password>'
+```
+
+```bash
+pip install browsergym-workarena
+playwright install
+workarena-install
+```
+
+
+</details>
+
 <details>
 <summary>WebArena on AWS</summary>
 TODO
@@ -65,17 +92,7 @@ TODO
 </details>
 
 
-<details>
-
-<summary>WorkArena</summary>
-
-```bash
-export SNOW_INSTANCE_URL="https://<your-instance-number>.service-now.com/"
-export SNOW_INSTANCE_UNAME="admin"
-export SNOW_INSTANCE_PWD=<your-instance-password>
-```
 
-</details>
 
 
 ## Launch experiments

reproducibility_journal.csv

@@ -1,5 +1,7 @@
-git_user,agent_name,benchmark,benchmark_version,date,avg_reward,std_err,n_err,n_completed,os,python_version,playwright_version,agentlab_version,agentlab_git_hash,agentlab__local_modifications,browsergym_version,browsergym_git_hash,browsergym__local_modifications
-recursix,GenericAgent-gpt-4o-mini-2024-07-18,miniwob_tiny_test,0.6.3,2024-09-19_21-07-34,0.75,0.217,0,4/4,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,c99bdf74c98f323cc6a646467ba5f21154b6fd18,,0.6.4,b73531271d2ce688c104eb4dfba2819583f1ba36,
-recursix,GenericAgent-gpt-4o-mini-2024-07-18,miniwob_tiny_test,0.6.3,2024-09-19_21-28-58,1.0,0.0,0,4/4,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,c99bdf74c98f323cc6a646467ba5f21154b6fd18,"  M: reproducibility_journal.csv
+git_user,agent_name,benchmark,benchmark_version,date,avg_reward,std_err,n_err,n_completed,comment,os,python_version,playwright_version,agentlab_version,agentlab_git_hash,agentlab__local_modifications,browsergym_version,browsergym_git_hash,browsergym__local_modifications
+recursix,GenericAgent-gpt-4o-mini-2024-07-18,miniwob_tiny_test,0.6.3,2024-09-19_21-07-34,0.75,0.217,0,4/4,,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,c99bdf74c98f323cc6a646467ba5f21154b6fd18,,0.6.4,b73531271d2ce688c104eb4dfba2819583f1ba36,
+recursix,GenericAgent-gpt-4o-mini-2024-07-18,miniwob_tiny_test,0.6.3,2024-09-19_21-28-58,1.0,0.0,0,4/4,,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,c99bdf74c98f323cc6a646467ba5f21154b6fd18,"  M: reproducibility_journal.csv
   M: src/agentlab/experiments/task_collections.py",0.6.4,b73531271d2ce688c104eb4dfba2819583f1ba36,
-recursix,GenericAgent-gpt-4o-mini-2024-07-18,miniwob,0.6.3,2024-09-20_07-16-21,0.546,0.02,0,625/625,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,295f01005faf8f2c73a31be6a18cec19d563b54b,,0.6.4,b73531271d2ce688c104eb4dfba2819583f1ba36,
+recursix,GenericAgent-gpt-4o-mini-2024-07-18,miniwob,0.6.3,2024-09-20_07-16-21,0.546,0.02,0,625/625,,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,295f01005faf8f2c73a31be6a18cec19d563b54b,,0.6.4,b73531271d2ce688c104eb4dfba2819583f1ba36,
+recursix,GenericAgent-gpt-4o-2024-05-13,miniwob,0.6.3,2024-09-20_22-09-43,0.656,0.019,0,625/625,,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,f6216486d5faac2c8b3fb0a63e114e5a4bafde47,,0.6.4,8cef8fe34940ff490d0cc06b0c8f100180d09d43,
+recursix,GenericAgent-gpt-4o-2024-05-13,miniwob,0.6.3,2024-09-21_12-04-39,0.656,0.019,0,625/625,None,Darwin (Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64),3.12.2,1.39.0,0.2.1,fe561b93c5f053e9f9625358862f542523b5e14a,,0.7.0,ed6d6992ef64bfb91aca7002d33cb6ed5ec031ef,

src/agentlab/agents/dynamic_prompting.py

@@ -577,9 +577,14 @@ def _parse_answer(self, text_answer):
                     ans_dict = {"action": code, "parse_error": str(e)}
 
         try:
-            # just check if action can be mapped to python code but keep action as is
-            # the environment will be responsible for mapping it to python
-            self.action_set.to_python_code(ans_dict["action"])
+            if ans_dict["action"] == "None":
+                # Used by reproducibility agent for backward compatibility of
+                # traces missing LLM's response in chat messages.
+                ans_dict["action"] = None
+            else:
+                # just check if action can be mapped to python code but keep action as is
+                # the environment will be responsible for mapping it to python
+                self.action_set.to_python_code(ans_dict["action"])
         except Exception as e:
             raise ParseError(
                 f"Error while parsing action\n: {e}\n"

src/agentlab/agents/generic_agent/__init__.py

@@ -1,7 +1,7 @@
 from .agent_configs import (
     AGENT_3_5,
     AGENT_8B,
-    AGENT_70B,
+    AGENT_LLAMA3_70B,
     AGENT_CUSTOM,
     RANDOM_SEARCH_AGENT,
     AGENT_4o,
@@ -14,7 +14,7 @@
     "AGENT_4o",
     "AGENT_4o_MINI",
     "AGENT_4o_VISION",
-    "AGENT_70B",
+    "AGENT_LLAMA3_70B",
     "AGENT_8B",
     "RANDOM_SEARCH_AGENT",
     "AGENT_CUSTOM",

src/agentlab/agents/generic_agent/agent_configs.py

@@ -96,7 +96,7 @@
 )
 
 # llama3-70b default config
-FLAGS_70B = GenericPromptFlags(
+FLAGS_LLAMA3_70B = GenericPromptFlags(
     obs=dp.ObsFlags(
         use_html=False,
         use_ax_tree=True,
@@ -135,9 +135,13 @@
     add_missparsed_messages=True,
 )
 
-AGENT_70B = GenericAgentArgs(
-    chat_model_args=CHAT_MODEL_ARGS_DICT["meta-llama/Meta-Llama-3-70B-Instruct"],
-    flags=FLAGS_70B,
+AGENT_LLAMA3_70B = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["openrouter/meta-llama/llama-3-70b-instruct"],
+    flags=FLAGS_LLAMA3_70B,
+)
+AGENT_LLAMA31_70B = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["openrouter/meta-llama/llama-3.1-70b-instruct"],
+    flags=FLAGS_LLAMA3_70B,
 )
 
 FLAGS_8B = GenericPromptFlags(
@@ -208,8 +212,8 @@
     action=dp.ActionFlags(
         multi_actions=False,
         action_set="bid",
-        long_description=True,
-        individual_examples=True,
+        long_description=False,
+        individual_examples=False,
     ),
     use_plan=False,
     use_criticise=False,

src/agentlab/agents/generic_agent/generic_agent.py

@@ -27,6 +27,7 @@ def __post_init__(self):
             pass
 
     def set_benchmark(self, benchmark):
+        """Override Some flags based on the benchmark."""
         if benchmark == "miniwob":
             self.flags.obs.use_html = True