Adding weblinx config to DEFAULT_BENCHMARKS

Makefile

@@ -1,12 +1,12 @@
 install:
 	@echo "--- 🚀 Installing project dependencies ---"
 	pip install -e ./browsergym/core -e ./browsergym/miniwob -e ./browsergym/webarena -e ./browsergym/visualwebarena/ -e ./browsergym/experiments -e ./browsergym/assistantbench -e ./browsergym/
-	playwright install chromium --with-deps
+	playwright install chromium
 
 install-demo:
 	@echo "--- 🚀 Installing demo dependencies ---"
 	pip install -r demo_agent/requirements.txt
-	playwright install chromium --with-deps
+	playwright install chromium
 
 demo:
 	@echo "--- 🚀 Running demo agent ---"

browsergym/core/src/browsergym/core/action/highlevel.py

@@ -195,6 +195,23 @@
         send_msg_to_user,
         report_infeasible,
     ],
+    # from weblinx_browsergym
+    # https://github.com/McGill-NLP/agentlab-weblinx-mvp/blob/a91b6d19870c5187d252e70a2e2013511cc6f1d2/weblinx_browsergym/__init__.py#L274-L286
+    "weblinx": [
+        send_msg_to_user,  # say(speaker="assistant", utterance=[str]) -> send_msg_to_user(text=[str])
+        click,  # click(uid=[element id]) -> click(bid=[element id])
+        hover,  # hover(uid=[element id]) -> hover(bid=[element id])
+        fill,  # textinput(uid=[element id], value=[str]) -> fill(bid=[element id], value=[str])
+        # change(uid=[element], value=[str]) -> ❌
+        goto,  # load(url=[link]) -> goto(url=[link])
+        # submit(uid=[element]) -> click(bid=[element id])
+        scroll,  # scroll(x=[int x],y=[int y]) -> scroll(delta_x=[int x], delta_y=[int y])
+        # copy(uid=[element],text=[str]) -> ❌
+        # paste(uid=[element],text=[str]) -> ❌
+        new_tab,  # tabcreate() -> new_tab()
+        tab_close,  # tabremove(target=[tabId]) -> tab_close()
+        tab_focus,  # tabswitch(origin=[origin tabId],target=[target tabId]) -> tab_focus(index=[target tabid])
+    ],
 }
 
 
@@ -224,6 +241,7 @@ class HighLevelActionSet(AbstractActionSet):
         "visualwebarena",
         "workarena",
         "workarena++",
+        "weblinx",
         "custom",
     ]
     DemoMode = typing.Literal["off", "default", "all_blue", "only_visible_elements"]

browsergym/experiments/src/browsergym/experiments/benchmark/__init__.py

@@ -1 +1,2 @@
-from .base import DEFAULT_BENCHMARKS, Benchmark, HighLevelActionSetArgs
+from .base import Benchmark, HighLevelActionSetArgs
+from .configs import DEFAULT_BENCHMARKS

browsergym/experiments/src/browsergym/experiments/benchmark/base.py

@@ -4,18 +4,13 @@
 from dataclasses import dataclass, field
 from typing import Literal, Optional
 
-import numpy as np
 import pandas as pd
 from dataclasses_json import DataClassJsonMixin, config
 
 from browsergym.core.action.highlevel import HighLevelActionSet
 from browsergym.experiments.loop import EnvArgs
 
-from .metadata.utils import task_list_from_metadata, task_metadata
-from .utils import (
-    make_env_args_list_from_repeat_tasks,
-    make_env_args_list_from_workarena_curriculum,
-)
+from .metadata.utils import task_list_from_metadata
 
 logger = logging.getLogger(__name__)
 
@@ -50,7 +45,9 @@ def make_action_set(self):
         )
 
 
-BenchmarkBackend = Literal["miniwob", "webarena", "visualwebarena", "workarena", "assistantbench"]
+BenchmarkBackend = Literal[
+    "miniwob", "webarena", "visualwebarena", "workarena", "assistantbench", "weblinx"
+]
 
 
 @dataclass
@@ -127,6 +124,10 @@ def prepare_backends(self):
                     # register environments
                     import browsergym.assistantbench
 
+                case "weblinx":
+                    # register environments
+                    import weblinx_browsergym
+
                 case _:
                     raise ValueError(f"Unknown benchmark backend {repr(backend)}")
 
@@ -170,188 +171,3 @@ def subset_from_regexp(self, column, regexp):
             ],
             task_metadata=self.task_metadata,
         )
-
-
-# These are mean as the default highlevel action set to fairly evaluate agents on each benchmark.
-# They are mostly arbitrary, the important thing is to evaluate different agents using the same action set for fairness.
-DEFAULT_HIGHLEVEL_ACTION_SET_ARGS = {
-    "miniwob_all": HighLevelActionSetArgs(
-        subsets=["miniwob_all"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    "miniwob_liu18": HighLevelActionSetArgs(
-        subsets=["miniwob_liu18"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    "miniwob_shi17": HighLevelActionSetArgs(
-        subsets=["miniwob_shi17"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    "miniwob_humphreys22": HighLevelActionSetArgs(
-        subsets=["miniwob_humphreys22"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    "workarena": HighLevelActionSetArgs(
-        subsets=["workarena"],  # no need for infeasible action
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    "workarena++": HighLevelActionSetArgs(
-        subsets=["workarena++"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    # from https://arxiv.org/abs/2307.13854
-    "webarena": HighLevelActionSetArgs(
-        subsets=["webarena"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    # from https://arxiv.org/abs/2401.13649
-    "visualwebarena": HighLevelActionSetArgs(
-        subsets=["visualwebarena"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-    "assistantbench": HighLevelActionSetArgs(
-        subsets=["chat", "bid", "tab", "nav"],
-        multiaction=False,
-        strict=False,
-        retry_with_force=True,
-        demo_mode="off",
-    ),
-}
-
-# all benchmarks are callables designed for lazy loading, i.e. `bench = DEFAULT_BENCHMARKS["miniwob_all"]()`
-DEFAULT_BENCHMARKS = {
-    "miniwob": lambda: Benchmark(
-        name="miniwob",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob_all"],
-        is_multi_tab=False,
-        backends=["miniwob"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("miniwob")),
-            max_steps=10,
-            n_repeats=5,
-            seeds_rng=np.random.RandomState(42),
-        ),
-        task_metadata=task_metadata("miniwob"),
-    ),
-    "miniwob_tiny_test": lambda: Benchmark(
-        name="miniwob_tiny_test",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob_all"],
-        is_multi_tab=False,
-        backends=["miniwob"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=["miniwob.click-dialog", "miniwob.click-checkboxes"],
-            max_steps=5,
-            n_repeats=2,
-            seeds_rng=np.random.RandomState(42),
-        ),
-        task_metadata=task_metadata("miniwob"),
-    ),
-    "webarena": lambda: Benchmark(
-        name="webarena",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
-        is_multi_tab=True,
-        backends=["webarena"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("webarena")),
-            max_steps=15,
-            n_repeats=1,
-            seeds_rng=np.random.RandomState(42),
-        ),
-        task_metadata=task_metadata("webarena"),
-    ),
-    "visualwebarena": lambda: Benchmark(
-        name="visualwebarena",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["visualwebarena"],
-        is_multi_tab=True,
-        backends=["visualwebarena"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("visualwebarena")),
-            max_steps=15,
-            n_repeats=1,
-            seeds_rng=np.random.RandomState(42),
-        ),
-        task_metadata=task_metadata("visualwebarena"),
-    ),
-    "workarena_l1": lambda: Benchmark(
-        name="workarena_l1",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena"],
-        is_multi_tab=False,
-        backends=["workarena"],
-        env_args_list=make_env_args_list_from_workarena_curriculum(
-            level="l1",
-            task_category_filter=None,
-            meta_seed=42,  # meta seed for evaluation curriculum
-            max_steps=15,
-            curriculum_type="agent",
-            seeds_l1=10,
-        ),
-        task_metadata=task_metadata("workarena"),
-    ),
-    "workarena_l2_agent_curriculum_eval": lambda: Benchmark(
-        name="workarena_l2_agent_curriculum_eval",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena++"],
-        is_multi_tab=True,
-        backends=["workarena"],
-        env_args_list=make_env_args_list_from_workarena_curriculum(
-            level="l2",
-            task_category_filter=None,
-            meta_seed=42,  # meta seed for evaluation curriculum
-            max_steps=50,
-            curriculum_type="agent",
-        ),
-        task_metadata=task_metadata("workarena"),
-    ),
-    "workarena_l3_agent_curriculum_eval": lambda: Benchmark(
-        name="workarena_l3_agent_curriculum_eval",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena++"],
-        is_multi_tab=True,
-        backends=["workarena"],
-        env_args_list=make_env_args_list_from_workarena_curriculum(
-            level="l3",
-            task_category_filter=None,
-            meta_seed=42,  # meta seed for evaluation curriculum
-            max_steps=50,
-            curriculum_type="agent",
-        ),
-        task_metadata=task_metadata("workarena"),
-    ),
-    "assistantbench": lambda: Benchmark(
-        name="assistantbench",
-        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["assistantbench"],
-        is_multi_tab=True,
-        backends=["assistantbench"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(
-                metadata=task_metadata("assistantbench"), filter={"browsergym_split": "valid|test"}
-            ),
-            max_steps=15,
-            n_repeats=1,
-            seeds_rng=np.random.RandomState(42),
-        ),
-        task_metadata=task_metadata("assistantbench"),
-    ),
-}