        # if "depends_on" column is missing, raise a warning and deal with it
        # (we don't want the "depends_on" column to be mandatory)
        if "depends_on" not in self.task_metadata.columns:
            logger.warning(
                f'This benchmark does not provide a dependency graph (missing a "depends_on" column in task metadata). Assuming no task dependencies.'
            )
            zero_dependencies = {task_name: [] for task_name in task_names}
            return zero_dependencies

        # recover the task dependency graph, for tasks in the benchmark only
        task_dependencies = extract_sparse_task_dependency_graph_from_subset(
            task_subset=task_names,
            parents=build_full_task_dependency_graph_from_metadata(
                task_metadata=self.task_metadata
            ),
        )

        return task_dependencies

    def dependency_graphs_over_env_args(self) -> list[dict[str, list[str]]]:
        """
        Returns a list of dependency graphs to be executed sequentially, typically with a full instance reset in-between.
        Ideally, a job scheduler should connect these graphs by injecting a reset task in-between each, which depends on all previous tasks being completed.
        """
        task_dependencies = self.dependency_graph_over_tasks()
        env_args_dependencies = build_env_args_dependency_graphs(
            env_args_list=self.env_args_list,
            task_dependencies=task_dependencies,
            supports_parallel_seeds=self.supports_parallel_seeds,
        )
        return env_args_dependencies


import numpy as np
from browsergym.experiments.benchmark.metadata.utils import (
    task_list_from_metadata,
    task_metadata,
)
from browsergym.experiments.benchmark.utils import (
    make_env_args_list_from_fixed_seeds,
    make_env_args_list_from_repeat_tasks,
    make_env_args_list_from_workarena_curriculum,
)

from .base import Benchmark, HighLevelActionSetArgs

# These are mean as the default highlevel action set to fairly evaluate agents on each benchmark.
# They are mostly arbitrary, the important thing is to evaluate different agents using the same action set for fairness.
DEFAULT_HIGHLEVEL_ACTION_SET_ARGS = {
    "miniwob_all": HighLevelActionSetArgs(
        subsets=["miniwob_all"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "miniwob_liu18": HighLevelActionSetArgs(
        subsets=["miniwob_liu18"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "miniwob_shi17": HighLevelActionSetArgs(
        subsets=["miniwob_shi17"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "miniwob_humphreys22": HighLevelActionSetArgs(
        subsets=["miniwob_humphreys22"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "workarena": HighLevelActionSetArgs(
        subsets=["workarena"],  # no need for infeasible action
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "workarena++": HighLevelActionSetArgs(
        subsets=["workarena++"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    # from https://arxiv.org/abs/2307.13854
    "webarena": HighLevelActionSetArgs(
        subsets=["webarena"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    # from https://arxiv.org/abs/2401.13649
    "visualwebarena": HighLevelActionSetArgs(
        subsets=["visualwebarena"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "assistantbench": HighLevelActionSetArgs(
        subsets=["assistantbench"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
    "weblinx": HighLevelActionSetArgs(
        subsets=["weblinx"],
        multiaction=False,
        strict=False,
        retry_with_force=True,
        demo_mode="off",
    ),
}

# all benchmarks are callables designed for lazy loading, i.e. `bench = DEFAULT_BENCHMARKS["miniwob_all"]()`
DEFAULT_BENCHMARKS = {
    "miniwob": lambda n_repeats=5: Benchmark(
        name="miniwob",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob_all"],
        is_multi_tab=False,
        supports_parallel_seeds=True,
        backends=["miniwob"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=task_list_from_metadata(metadata=task_metadata("miniwob")),
            max_steps=10,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("miniwob"),
    ),
    "miniwob_tiny_test": lambda n_repeats=2: Benchmark(
        name="miniwob_tiny_test",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob_all"],
        is_multi_tab=False,
        supports_parallel_seeds=True,
        backends=["miniwob"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=["miniwob.click-dialog", "miniwob.click-checkboxes"],
            max_steps=5,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("miniwob"),
    ),
    "webarena": lambda n_repeats=1: Benchmark(
        name="webarena",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
        is_multi_tab=True,
        supports_parallel_seeds=False,
        backends=["webarena"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=task_list_from_metadata(metadata=task_metadata("webarena")),
            max_steps=30,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("webarena"),
    ),
    "webarena_lite": lambda n_repeats=1: Benchmark(
        name="webarena_lite",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
        is_multi_tab=True,
        supports_parallel_seeds=False,
        backends=["webarena"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=task_list_from_metadata(metadata=task_metadata("webarenalite")),
            max_steps=30,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("webarenalite"),
    ),
    "webarena_tiny": lambda n_repeats=1: Benchmark(
        name="webarena_tiny",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
        is_multi_tab=True,
        supports_parallel_seeds=False,
        backends=["webarena"],
        env_args_list=make_env_args_list_from_fixed_seeds(
            task_list=[
                "webarena.410",
                "webarena.533",
                "webarena.537",
                "webarena.552",
                "webarena.561",
                "webarena.562",
            ],
            max_steps=30,
            fixed_seeds=[0],
        ),
        task_metadata=task_metadata("webarena"),
    ),
    "visualwebarena_tiny": lambda n_repeats=10: Benchmark(
        name="visualwebarena_tiny",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["visualwebarena"],
        is_multi_tab=True,
        supports_parallel_seeds=False,
        backends=["visualwebarena"],
        env_args_list=make_env_args_list_from_fixed_seeds(
            task_list=[
                "visualwebarena.228",
                "visualwebarena.263",
                "visualwebarena.550",
                "visualwebarena.784",
            ],
            max_steps=30,
            fixed_seeds=[0],
        ),
        task_metadata=task_metadata("visualwebarena"),
    ),
    "visualwebarena": lambda n_repeats=1: Benchmark(
        name="visualwebarena",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["visualwebarena"],
        is_multi_tab=True,
        supports_parallel_seeds=False,
        backends=["visualwebarena"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=task_list_from_metadata(metadata=task_metadata("visualwebarena")),
            max_steps=30,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("visualwebarena"),
    ),
    "workarena_l1": lambda n_repeats=10: Benchmark(
        name="workarena_l1",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena"],
        is_multi_tab=False,
        supports_parallel_seeds=True,
        backends=["workarena"],
        env_args_list=make_env_args_list_from_workarena_curriculum(
            level="l1",
            task_category_filter=None,
            meta_seed=42,  # meta seed for evaluation curriculum
            max_steps=15,
            curriculum_type="agent",
            seeds_l1=n_repeats,
        ),
        task_metadata=task_metadata("workarena"),
    ),
    "workarena_l2_agent_curriculum_eval": lambda n_repeats=1: Benchmark(
        name="workarena_l2_agent_curriculum_eval",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena++"],
        is_multi_tab=True,
        supports_parallel_seeds=True,
        backends=["workarena"],
        env_args_list=make_env_args_list_from_workarena_curriculum(
            level="l2",
            task_category_filter=None,
            meta_seed=42,  # meta seed for evaluation curriculum
            max_steps=50,
            curriculum_type="agent",
        ),
        task_metadata=task_metadata("workarena"),
    ),
    "workarena_l3_agent_curriculum_eval": lambda n_repeats=1: Benchmark(
        name="workarena_l3_agent_curriculum_eval",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena++"],
        is_multi_tab=True,
        supports_parallel_seeds=True,
        backends=["workarena"],
        env_args_list=make_env_args_list_from_workarena_curriculum(
            level="l3",
            task_category_filter=None,
            meta_seed=42,  # meta seed for evaluation curriculum
            max_steps=50,
            curriculum_type="agent",
        ),
        task_metadata=task_metadata("workarena"),
    ),
    "assistantbench": lambda n_repeats=1: Benchmark(
        name="assistantbench",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["assistantbench"],
        is_multi_tab=True,
        supports_parallel_seeds=True,
        backends=["assistantbench"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=task_list_from_metadata(
                metadata=task_metadata("assistantbench"), filter={"browsergym_split": "valid|test"}
            ),
            max_steps=30,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("assistantbench"),
    ),
    "weblinx": lambda n_repeats=1: Benchmark(
        name="weblinx",
        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["weblinx"],
        is_multi_tab=True,
        supports_parallel_seeds=True,
        backends=["weblinx"],
        env_args_list=make_env_args_list_from_repeat_tasks(
            task_list=task_list_from_metadata(metadata=task_metadata("weblinx")),
            max_steps=1,
            n_repeats=n_repeats,
            seeds_rng=np.random.RandomState(42),
        ),
        task_metadata=task_metadata("weblinx"),
    ),
}


import logging
import multiprocessing as mp
import os
import traceback
import typing
from typing import Literal

import numpy as np
from browsergym.experiments.loop import SEED_MAX, EnvArgs

logger = logging.getLogger(__name__)

def make_env_args_list_from_workarena_curriculum(
    level: Literal["l1", "l2", "l3"],
    task_category_filter: str,
    meta_seed: int,
    max_steps: int,
    curriculum_type: Literal["human", "agent"],
    seeds_l1: int = 10,
):
    """
    Returns a WorkArena predefined task curriculum (e.g., task and seed combination).
    """
    assert level in ("l1", "l2", "l3")
    assert curriculum_type in ("human", "agent")

    env_args_list = []

    # dynamic import
    from browsergym.workarena import get_all_tasks_agents

    all_task_tuples = get_all_tasks_agents(
        filter=f"{level}.{task_category_filter}" if task_category_filter else level,
        meta_seed=meta_seed,
        is_agent_curriculum=(curriculum_type == "agent"),
        n_seed_l1=seeds_l1,
    )

    for task, seed in all_task_tuples:
        task_name = task.get_task_id()
        env_args_list.append(EnvArgs(task_name=task_name, task_seed=seed, max_steps=max_steps))

    return env_args_list

def make_env_args_list_from_repeat_tasks(
    task_list: list[str],
    max_steps: int,
    n_repeats: int,
    seeds_rng: np.random.RandomState,
    viewport=None,
):
    """
    Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
    """
    env_args_list = []
    for task in task_list:
        for seed in seeds_rng.randint(low=0, high=SEED_MAX, size=n_repeats):
            env_args_list.append(
                EnvArgs(
                    task_name=task,
                    task_seed=int(seed),
                    max_steps=max_steps,
                    headless=True,
                    record_video=False,
                    wait_for_user_message=False,
                    viewport=viewport,
                    slow_mo=None,
                    storage_state=None,
                    task_kwargs=None,
                )
            )

    return env_args_list

def make_env_args_list_from_fixed_seeds(
    task_list: list[str], max_steps: int, fixed_seeds: list[int], n_repeats: int = 1
):
    """
    Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
    """
    env_args_list = []
    for task in task_list:
        for seed in fixed_seeds:
            for _ in range(n_repeats):
                env_args_list.append(
                    EnvArgs(
                        task_name=task,
                        task_seed=int(seed),
                        max_steps=max_steps,
                        headless=True,
                        record_video=False,
                        wait_for_user_message=False,
                        viewport=None,
                        slow_mo=None,
                        storage_state=None,
                        task_kwargs=None,
                    )
                )

    return env_args_list

def prepare_backend(backend: str):
    match backend:
        case "miniwob":
            # register environments
            import browsergym.miniwob

            # check setup
            browsergym.miniwob.environment_variables_precheck()

        case "webarena":
            # register environments
            import browsergym.webarena

            # full reset the instance (requires environment variables properly set up)
            from browsergym.webarena.instance import WebArenaInstance

            default_instance = WebArenaInstance()
            default_instance.full_reset()

            logging.info(
                f"Initiating WebArena instance warm-up. Some tasks will be pre-loaded (massaged) to trigger some caching mechanisms and make the server more responsive."
            )
            massage_tasks(
                [
                    f"webarena.{id}"
                    for id in [
                        410,  # reddit
                        533,  # gitlab
                        561,  # gitlab wiki
                        562,  # gitlab reddit
                        574,  # shopping
                        640,  # reddit
                        680,  # shopping_admin
                        740,  # wiki map
                    ]
                ]
            )

        case "visualwebarena":
            # register environments
            import browsergym.visualwebarena

            # full reset the instance (requires environment variables properly set up)
            from browsergym.visualwebarena.instance import VisualWebArenaInstance

            default_instance = VisualWebArenaInstance()
            default_instance.full_reset()

            logging.info(
                f"Initiating VisualWebArena instance warm-up. Some tasks will be pre-loaded (massaged) to trigger some caching mechanisms and make the server more responsive."
            )
            massage_tasks(
                [
                    f"visualwebarena.{id}"
                    for id in [
                        0,  # classifieds
                        33,  # classifieds
                        555,  # shopping
                        666,  # shopping
                        282,  # __REDDIT__/f/dataisbeautiful
                        305,  # __REDDIT__/f/memes/new
                        314,  # __REDDIT__/f/mildlyinteresting
                        317,  # __REDDIT__/f/Art/active
                        318,  # __REDDIT__/f/consoles
                        319,  # __REDDIT__/f/EarthPorn
                        410,  # __REDDIT__/f/food
                        411,  # __REDDIT__/f/food
                        427,  # __REDDIT__/f/EarthPorn
                        436,  # __REDDIT__/f/Art
                        440,  # __REDDIT__/f/EarthPorn
                    ]
                ]
            )

        case "workarena":
            # register environments
            import browsergym.workarena

            # check server status
            from browsergym.workarena.instance import SNowInstance

            default_instance = SNowInstance()
            default_instance.check_status()

        case "assistantbench":
            # register environments
            import browsergym.assistantbench

        case "weblinx":
            # register environments
            import weblinx_browsergym

            # pre-download all weblinx files
            cache_dir = os.environ.get("BROWSERGYM_WEBLINX_CACHE_DIR", None)

            assert (
                cache_dir
            ), f"Environment variable BROWSERGYM_WEBLINX_CACHE_DIR is missing or empty, required to prepare the weblinx backend."

            all_tasks = []
            for split in ("train", "valid", "test_iid"):
                all_tasks.extend(weblinx_browsergym.list_tasks(split=split, cache_dir=cache_dir))
            demo_ids = weblinx_browsergym.get_unique_demo_ids(tasks=all_tasks)
            weblinx_browsergym.download_and_unzip_demos(demo_ids=demo_ids, cache_dir=cache_dir)

        case _:
            raise NotImplementedError(f"Unknown benchmark backend {repr(backend)}")

def massage_tasks(task_ids: list[str], max_retries: int = 1, timeout: int = 60):
    for i, task_id in enumerate(task_ids):
        logger.info(f"Massaging task {i + 1} / {len(task_ids)}: {task_id}")
        for retries in range(max_retries + 1):
            outcome, err_msg = massage_task_within_subprocess(task_id=task_id, timeout=timeout)
            if outcome == "success":
                break
            if retries < max_retries:
                logger.info(
                    f"Massage resulted in {outcome}, retrying ({retries + 1} / {max_retries} retries)"
                )
            else:
                logger.warning(
                    f"Massage unsuccessful after {retries} retries, skipping. Last error message: {err_msg}"
                )

def run_massage(outcome_queue: mp.Queue, task_id: str):
    import gymnasium as gym

    gym_id = f"browsergym/{task_id}"
    env = gym.make(gym_id)
    no_action = "noop()"
    # check if action space exists and is compatible with "noop()"
    try:
        env.unwrapped.action_mapping(no_action)
    except:
        no_action = ""  # fallback plan
    # run massage
    try:
        env.reset()  # task setup
        env.step(no_action)  # task validation
        env.step(no_action)  # task validation again
        outcome = "success", None
    except Exception as e:
        outcome = "exception", traceback.format_exception(e)
    finally:
        env.close()
        outcome_queue.put(outcome)

def massage_task_within_subprocess(
    task_id: str, timeout: int, kill_timeout: int = 10
) -> typing.Tuple[str, str]:
    """Massages a BrowserGym task (reset, noop, noop) inside a subprocess to monitor execution
    times and kill the process after a timeout.

    Returns: an (outcome, err_msg) tuple.
      - outcome: the outcome of the massage, one of 'success', 'exception' or 'timeout'.
      - err_msg: error message if any, or None.
    """

    queue = mp.Queue()
    process = mp.Process(target=run_massage, args=(queue, task_id))
    process.start()
    process.join(timeout=timeout)

    if process.is_alive():
        # if the process is still alive after the timeout
        outcome = "timeout", f"Timeout {timeout} seconds exceeded"
        process.kill()
        process.join(timeout=kill_timeout)
        if process.is_alive():
            # if the process is still alive after the kill
            logger.warning(
                f"Massage sub-process still alive {kill_timeout} seconds after kill(), you might have a zombie process now."
            )
    else:
        outcome = queue.get_nowait()

    return outcome


import logging
import multiprocessing as mp
import os
import traceback
import typing
from typing import Literal

import numpy as np
from browsergym.experiments.loop import SEED_MAX, EnvArgs

logger = logging.getLogger(__name__)

def make_env_args_list_from_workarena_curriculum(
    level: Literal["l1", "l2", "l3"],
    task_category_filter: str,
    meta_seed: int,
    max_steps: int,
    curriculum_type: Literal["human", "agent"],
    seeds_l1: int = 10,
):
    """
    Returns a WorkArena predefined task curriculum (e.g., task and seed combination).
    """
    assert level in ("l1", "l2", "l3")
    assert curriculum_type in ("human", "agent")

    env_args_list = []

    # dynamic import
    from browsergym.workarena import get_all_tasks_agents

    all_task_tuples = get_all_tasks_agents(
        filter=f"{level}.{task_category_filter}" if task_category_filter else level,
        meta_seed=meta_seed,
        is_agent_curriculum=(curriculum_type == "agent"),
        n_seed_l1=seeds_l1,
    )

    for task, seed in all_task_tuples:
        task_name = task.get_task_id()
        env_args_list.append(EnvArgs(task_name=task_name, task_seed=seed, max_steps=max_steps))

    return env_args_list

def make_env_args_list_from_repeat_tasks(
    task_list: list[str],
    max_steps: int,
    n_repeats: int,
    seeds_rng: np.random.RandomState,
    viewport=None,
):
    """
    Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
    """
    env_args_list = []
    for task in task_list:
        for seed in seeds_rng.randint(low=0, high=SEED_MAX, size=n_repeats):
            env_args_list.append(
                EnvArgs(
                    task_name=task,
                    task_seed=int(seed),
                    max_steps=max_steps,


    env_args_list = []

    # dynamic import
    from browsergym.workarena import get_all_tasks_agents

    all_task_tuples = get_all_tasks_agents(
        filter=f"{level}.{task_category_filter}" if task_category_filter else level,
        meta_seed=meta_seed,
        is_agent_curriculum=(curriculum_type == "agent"),
        n_seed_l1=seeds_l1,
    )

    for task, seed in all_task_tuples:
        task_name = task.get_task_id()
        env_args_list.append(EnvArgs(task_name=task_name, task_seed=seed, max_steps=max_steps))

    return env_args_list

def make_env_args_list_from_repeat_tasks(
    task_list: list[str],
    max_steps: int,
    n_repeats: int,
    seeds_rng: np.random.RandomState,
    viewport=None,
):
    """
    Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
    """
    env_args_list = []
    for task in task_list:
        for seed in seeds_rng.randint(low=0, high=SEED_MAX, size=n_repeats):
            env_args_list.append(
                EnvArgs(
                    task_name=task,
                    task_seed=int(seed),
                    max_steps=max_steps,
                    headless=True,
                    record_video=False,
                    wait_for_user_message=False,
                    viewport=viewport,
                    slow_mo=None,
                    storage_state=None,
                    task_kwargs=None,
                )
            )

    return env_args_list

def make_env_args_list_from_fixed_seeds(
    task_list: list[str], max_steps: int, fixed_seeds: list[int], n_repeats: int = 1
):
    """
    Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
    """
    env_args_list = []
    for task in task_list:
        for seed in fixed_seeds:
            for _ in range(n_repeats):
                env_args_list.append(
                    EnvArgs(
                        task_name=task,
                        task_seed=int(seed),
                        max_steps=max_steps,
                        headless=True,
                        record_video=False,
                        wait_for_user_message=False,


    for task in task_list:
        for seed in seeds_rng.randint(low=0, high=SEED_MAX, size=n_repeats):
            env_args_list.append(
                EnvArgs(
                    task_name=task,
                    task_seed=int(seed),
                    max_steps=max_steps,
                    headless=True,
                    record_video=False,
                    wait_for_user_message=False,
                    viewport=viewport,
                    slow_mo=None,
                    storage_state=None,
                    task_kwargs=None,
                )
            )

    return env_args_list

def make_env_args_list_from_fixed_seeds(
    task_list: list[str], max_steps: int, fixed_seeds: list[int], n_repeats: int = 1
):
    """
    Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
    """
    env_args_list = []
    for task in task_list:
        for seed in fixed_seeds:
            for _ in range(n_repeats):
                env_args_list.append(
                    EnvArgs(
                        task_name=task,
                        task_seed=int(seed),
                        max_steps=max_steps,
                        headless=True,
                        record_video=False,
                        wait_for_user_message=False,
                        viewport=None,
                        slow_mo=None,
                        storage_state=None,
                        task_kwargs=None,
                    )
                )

    return env_args_list

def prepare_backend(backend: str):
    match backend:
        case "miniwob":
            # register environments
            import browsergym.miniwob

            # check setup
            browsergym.miniwob.environment_variables_precheck()

        case "webarena":
            # register environments
            import browsergym.webarena

            # full reset the instance (requires environment variables properly set up)
            from browsergym.webarena.instance import WebArenaInstance

            default_instance = WebArenaInstance()
            default_instance.full_reset()


        for seed in fixed_seeds:
            for _ in range(n_repeats):
                env_args_list.append(
                    EnvArgs(
                        task_name=task,
                        task_seed=int(seed),
                        max_steps=max_steps,
                        headless=True,
                        record_video=False,
                        wait_for_user_message=False,
                        viewport=None,
                        slow_mo=None,
                        storage_state=None,
                        task_kwargs=None,
                    )
                )

    return env_args_list

def prepare_backend(backend: str):
    match backend:
        case "miniwob":
            # register environments
            import browsergym.miniwob

            # check setup
            browsergym.miniwob.environment_variables_precheck()

        case "webarena":
            # register environments
            import browsergym.webarena

            # full reset the instance (requires environment variables properly set up)
            from browsergym.webarena.instance import WebArenaInstance

            default_instance = WebArenaInstance()
            default_instance.full_reset()

            logging.info(
                f"Initiating WebArena instance warm-up. Some tasks will be pre-loaded (massaged) to trigger some caching mechanisms and make the server more responsive."
            )
            massage_tasks(
                [
                    f"webarena.{id}"
                    for id in [
                        410,  # reddit
                        533,  # gitlab
                        561,  # gitlab wiki
                        562,  # gitlab reddit
                        574,  # shopping
                        640,  # reddit
                        680,  # shopping_admin
                        740,  # wiki map
                    ]
                ]
            )

        case "visualwebarena":
            # register environments
            import browsergym.visualwebarena

            # full reset the instance (requires environment variables properly set up)
            from browsergym.visualwebarena.instance import VisualWebArenaInstance

            default_instance = VisualWebArenaInstance()
            default_instance.full_reset()

            logging.info(
                f"Initiating VisualWebArena instance warm-up. Some tasks will be pre-loaded (massaged) to trigger some caching mechanisms and make the server more responsive."
            )
            massage_tasks(
                [
                    f"visualwebarena.{id}"
                    for id in [
                        0,  # classifieds
                        33,  # classifieds
                        555,  # shopping
                        666,  # shopping
                        282,  # __REDDIT__/f/dataisbeautiful
                        305,  # __REDDIT__/f/memes/new
                        314,  # __REDDIT__/f/mildlyinteresting
                        317,  # __REDDIT__/f/Art/active
                        318,  # __REDDIT__/f/consoles
                        319,  # __REDDIT__/f/EarthPorn
                        410,  # __REDDIT__/f/food
                        411,  # __REDDIT__/f/food
                        427,  # __REDDIT__/f/EarthPorn
                        436,  # __REDDIT__/f/Art
                        440,  # __REDDIT__/f/EarthPorn
                    ]
                ]
            )

        case "workarena":
            # register environments
            import browsergym.workarena

            # check server status
            from browsergym.workarena.instance import SNowInstance

            default_instance = SNowInstance()
            default_instance.check_status()

        case "assistantbench":
            # register environments
            import browsergym.assistantbench

        case "weblinx":
            # register environments
            import weblinx_browsergym

            # pre-download all weblinx files
            cache_dir = os.environ.get("BROWSERGYM_WEBLINX_CACHE_DIR", None)

            assert (
                cache_dir
            ), f"Environment variable BROWSERGYM_WEBLINX_CACHE_DIR is missing or empty, required to prepare the weblinx backend."

            all_tasks = []
            for split in ("train", "valid", "test_iid"):
                all_tasks.extend(weblinx_browsergym.list_tasks(split=split, cache_dir=cache_dir))
            demo_ids = weblinx_browsergym.get_unique_demo_ids(tasks=all_tasks)
            weblinx_browsergym.download_and_unzip_demos(demo_ids=demo_ids, cache_dir=cache_dir)

        case _:
            raise NotImplementedError(f"Unknown benchmark backend {repr(backend)}")

def massage_tasks(task_ids: list[str], max_retries: int = 1, timeout: int = 60):
    for i, task_id in enumerate(task_ids):
        logger.info(f"Massaging task {i + 1} / {len(task_ids)}: {task_id}")
        for retries in range(max_retries + 1):
            outcome, err_msg = massage_task_within_subprocess(task_id=task_id, timeout=timeout)
            if outcome == "success":
                break
            if retries < max_retries:
                logger.info(
                    f"Massage resulted in {outcome}, retrying ({retries + 1} / {max_retries} retries)"
                )
            else:
                logger.warning(
                    f"Massage unsuccessful after {retries} retries, skipping. Last error message: {err_msg}"
                )

def run_massage(outcome_queue: mp.Queue, task_id: str):


            # register environments
            import weblinx_browsergym

            # pre-download all weblinx files
            cache_dir = os.environ.get("BROWSERGYM_WEBLINX_CACHE_DIR", None)

            assert (
                cache_dir
            ), f"Environment variable BROWSERGYM_WEBLINX_CACHE_DIR is missing or empty, required to prepare the weblinx backend."

            all_tasks = []
            for split in ("train", "valid", "test_iid"):
                all_tasks.extend(weblinx_browsergym.list_tasks(split=split, cache_dir=cache_dir))
            demo_ids = weblinx_browsergym.get_unique_demo_ids(tasks=all_tasks)
            weblinx_browsergym.download_and_unzip_demos(demo_ids=demo_ids, cache_dir=cache_dir)

        case _:
            raise NotImplementedError(f"Unknown benchmark backend {repr(backend)}")

def massage_tasks(task_ids: list[str], max_retries: int = 1, timeout: int = 60):
    for i, task_id in enumerate(task_ids):
        logger.info(f"Massaging task {i + 1} / {len(task_ids)}: {task_id}")
        for retries in range(max_retries + 1):
            outcome, err_msg = massage_task_within_subprocess(task_id=task_id, timeout=timeout)
            if outcome == "success":
                break
            if retries < max_retries:
                logger.info(
                    f"Massage resulted in {outcome}, retrying ({retries + 1} / {max_retries} retries)"
                )
            else:
                logger.warning(
                    f"Massage unsuccessful after {retries} retries, skipping. Last error message: {err_msg}"
                )

def run_massage(outcome_queue: mp.Queue, task_id: str):
    import gymnasium as gym

    gym_id = f"browsergym/{task_id}"
    env = gym.make(gym_id)
    no_action = "noop()"
    # check if action space exists and is compatible with "noop()"
    try:
        env.unwrapped.action_mapping(no_action)
    except:
        no_action = ""  # fallback plan
    # run massage
    try:
        env.reset()  # task setup
        env.step(no_action)  # task validation
        env.step(no_action)  # task validation again
        outcome = "success", None
    except Exception as e:


            raise NotImplementedError(f"Unknown benchmark backend {repr(backend)}")

def massage_tasks(task_ids: list[str], max_retries: int = 1, timeout: int = 60):
    for i, task_id in enumerate(task_ids):
        logger.info(f"Massaging task {i + 1} / {len(task_ids)}: {task_id}")
        for retries in range(max_retries + 1):
            outcome, err_msg = massage_task_within_subprocess(task_id=task_id, timeout=timeout)
            if outcome == "success":
                break
            if retries < max_retries:
                logger.info(
                    f"Massage resulted in {outcome}, retrying ({retries + 1} / {max_retries} retries)"
                )
            else:
                logger.warning(
                    f"Massage unsuccessful after {retries} retries, skipping. Last error message: {err_msg}"
                )

def run_massage(outcome_queue: mp.Queue, task_id: str):
    import gymnasium as gym

    gym_id = f"browsergym/{task_id}"
    env = gym.make(gym_id)
    no_action = "noop()"
    # check if action space exists and is compatible with "noop()"
    try:
        env.unwrapped.action_mapping(no_action)
    except:
        no_action = ""  # fallback plan
    # run massage
    try:
        env.reset()  # task setup
        env.step(no_action)  # task validation
        env.step(no_action)  # task validation again
        outcome = "success", None
    except Exception as e:
        outcome = "exception", traceback.format_exception(e)
    finally:
        env.close()
        outcome_queue.put(outcome)

def massage_task_within_subprocess(
    task_id: str, timeout: int, kill_timeout: int = 10
) -> typing.Tuple[str, str]:
    """Massages a BrowserGym task (reset, noop, noop) inside a subprocess to monitor execution
    times and kill the process after a timeout.

    Returns: an (outcome, err_msg) tuple.
      - outcome: the outcome of the massage, one of 'success', 'exception' or 'timeout'.
      - err_msg: error message if any, or None.
    """

    queue = mp.Queue()
    process = mp.Process(target=run_massage, args=(queue, task_id))
    process.start()
    process.join(timeout=timeout)

    if process.is_alive():
        # if the process is still alive after the timeout


    env = gym.make(gym_id)
    no_action = "noop()"
    # check if action space exists and is compatible with "noop()"
    try:
        env.unwrapped.action_mapping(no_action)
    except:
        no_action = ""  # fallback plan
    # run massage
    try:
        env.reset()  # task setup
        env.step(no_action)  # task validation
        env.step(no_action)  # task validation again
        outcome = "success", None
    except Exception as e:
        outcome = "exception", traceback.format_exception(e)
    finally:
        env.close()
        outcome_queue.put(outcome)

def massage_task_within_subprocess(
    task_id: str, timeout: int, kill_timeout: int = 10
) -> typing.Tuple[str, str]:
    """Massages a BrowserGym task (reset, noop, noop) inside a subprocess to monitor execution
    times and kill the process after a timeout.

    Returns: an (outcome, err_msg) tuple.
      - outcome: the outcome of the massage, one of 'success', 'exception' or 'timeout'.
      - err_msg: error message if any, or None.
    """

    queue = mp.Queue()
    process = mp.Process(target=run_massage, args=(queue, task_id))
    process.start()
    process.join(timeout=timeout)

    if process.is_alive():
        # if the process is still alive after the timeout
        outcome = "timeout", f"Timeout {timeout} seconds exceeded"
        process.kill()
        process.join(timeout=kill_timeout)
        if process.is_alive():
            # if the process is still alive after the kill
            logger.warning(
                f"Massage sub-process still alive {kill_timeout} seconds after kill(), you might have a zombie process now."
            )
    else:
        outcome = queue.get_nowait()

    return outcome


import io
import pkgutil
from collections import defaultdict
from copy import deepcopy
from typing import Any, Optional

import pandas as pd

from browsergym.experiments.loop import EnvArgs

def task_metadata(benchmark_name: str):
    return task_metadata_from_csv(
        io.StringIO(pkgutil.get_data(__name__, f"{benchmark_name}.csv").decode("utf-8"))
    )

def task_metadata_from_csv(filepath):
    return pd.read_csv(filepath).fillna("")

def task_list_from_metadata(metadata: pd.DataFrame, filter: dict[str, str] = {}):
    df = metadata
    # filter the desired columns (AND filter)
    for col_name, regex in filter.items():
        col_filter = df[col_name].astype(str).str.contains(regex, regex=True)
        df = df[col_filter]
    # return only the task names
    return list(df["task_name"])

def build_full_task_dependency_graph_from_metadata(
    task_metadata: pd.DataFrame,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    if "depends_on" not in task_metadata.columns:
        raise ValueError(
            'task metadata column "depends_on" is required to compute task dependencies.'
        )

    graph_parents = {
        task_name: depends_on.split()
        for task_name, depends_on in zip(task_metadata["task_name"], task_metadata["depends_on"])
    }

    return graph_parents

def extract_graph_children(parents: dict[Any, list[Any]]):
    # start with empty children
    children = {task_name: [] for task_name in parents.keys()}
    # build children sets iteratively
    for child, parents in parents.items():
        for parent in parents:
            children[parent].append(child)
    return children

def extract_sparse_task_dependency_graph_from_subset(
    task_subset: list[str],
    parents: dict[str, list[str]],
    children: Optional[dict[str, list[str]]] = None,
    return_children: bool = False,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:

    if children is None:
        children = extract_graph_children(parents)

    # consistency check
    assert all([task in parents for task in task_subset])
    assert all([task in children for task in task_subset])

    # copy the graph
    subgraph_parents = deepcopy(parents)
    subgraph_children = deepcopy(children)

    # prune the graph (node contraction)
    for task_name in parents.keys():
        # if task is not present in the target task subset, drop its node
        if task_name not in task_subset:
            # connect node's children to node's parents
            for child in subgraph_children[task_name]:
                subgraph_parents[child].remove(task_name)
                subgraph_parents[child].extend(subgraph_parents[task_name])
            # connect node's parents to node's children
            for parent in subgraph_parents[task_name]:
                subgraph_children[parent].remove(task_name)
                subgraph_children[parent].extend(subgraph_children[task_name])
            # remove node
            del subgraph_parents[task_name]
            del subgraph_children[task_name]

    # return parent and (optionally) children mappings
    if return_children:
        return subgraph_parents, subgraph_children

    return subgraph_parents

def build_env_args_dependency_graphs(
    env_args_list: list[EnvArgs],
    task_dependencies: dict[str, list[str]],
    supports_parallel_seeds: bool,
) -> list[dict[int, list[int]]]:
    """
    Returns a list of dependency graphs to be executed sequentially, typically with a full instance reset in-between.
    Ideally, a job scheduler should connect these graphs by injecting a reset task in-between each, which depends on all previous tasks being completed.
    """
    # build mapping from tasks to run subsets (task_name -> list of env_args_list indices)
    task_runs = defaultdict(list)
    for i, env_args in enumerate(env_args_list):
        task_runs[env_args.task_name].append(i)
    task_runs = dict(task_runs)

    # consistency check
    assert all([task in task_dependencies for task in task_runs.keys()])

    # divide same-task runs into distinct splits if needed
    task_runs_splits = []
    if supports_parallel_seeds:
        # if parallel task runs over seeds is supported, do not split
        task_runs_splits.append(task_runs)
    else:
        # else, split runs so that each task has only one run (seed) in each split
        while task_runs:
            # extract first task run only (one seed per task)
            split = {task_name: [runs.pop(0)] for task_name, runs in task_runs.items()}
            task_runs_splits.append(split)
            # update task list to only those with remaining runs (seeds)
            task_runs = {task_name: runs for task_name, runs in task_runs.items() if runs}

    # recover the parent and child mappings of the task dependency graph
    task_parents = task_dependencies
    task_children = extract_graph_children(task_parents)

    # for each split, build the task dependency subgraph (task_name nodes), then build the run dependency graph (env_args index nodes)
    run_parents_split = []
    for split in task_runs_splits:
        # build the task dependency graph (task_name nodes)
        split_task_names = list(split.keys())
        split_task_parents = extract_sparse_task_dependency_graph_from_subset(
            task_subset=split_task_names,
            parents=task_parents,
            children=task_children,
        )
        # then, build the run dependency graph (env_args index nodes)
        split_run_parents = defaultdict(list)
        for task_name, runs in split.items():
            parent_tasks_runs = sum([split[parent] for parent in split_task_parents[task_name]], [])
            for run in runs:
                split_run_parents[run].extend(parent_tasks_runs)
        split_run_parents = dict(split_run_parents)
        run_parents_split.append(split_run_parents)

    return run_parents_split


import io
import pkgutil
from collections import defaultdict
from copy import deepcopy
from typing import Any, Optional

import pandas as pd

from browsergym.experiments.loop import EnvArgs

def task_metadata(benchmark_name: str):
    return task_metadata_from_csv(
        io.StringIO(pkgutil.get_data(__name__, f"{benchmark_name}.csv").decode("utf-8"))
    )

def task_metadata_from_csv(filepath):
    return pd.read_csv(filepath).fillna("")

def task_list_from_metadata(metadata: pd.DataFrame, filter: dict[str, str] = {}):
    df = metadata
    # filter the desired columns (AND filter)
    for col_name, regex in filter.items():
        col_filter = df[col_name].astype(str).str.contains(regex, regex=True)
        df = df[col_filter]
    # return only the task names
    return list(df["task_name"])

def build_full_task_dependency_graph_from_metadata(
    task_metadata: pd.DataFrame,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    if "depends_on" not in task_metadata.columns:


import io
import pkgutil
from collections import defaultdict
from copy import deepcopy
from typing import Any, Optional

import pandas as pd

from browsergym.experiments.loop import EnvArgs

def task_metadata(benchmark_name: str):
    return task_metadata_from_csv(
        io.StringIO(pkgutil.get_data(__name__, f"{benchmark_name}.csv").decode("utf-8"))
    )

def task_metadata_from_csv(filepath):
    return pd.read_csv(filepath).fillna("")

def task_list_from_metadata(metadata: pd.DataFrame, filter: dict[str, str] = {}):
    df = metadata
    # filter the desired columns (AND filter)
    for col_name, regex in filter.items():
        col_filter = df[col_name].astype(str).str.contains(regex, regex=True)
        df = df[col_filter]
    # return only the task names
    return list(df["task_name"])

def build_full_task_dependency_graph_from_metadata(
    task_metadata: pd.DataFrame,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    if "depends_on" not in task_metadata.columns:
        raise ValueError(
            'task metadata column "depends_on" is required to compute task dependencies.'
        )


import pkgutil
from collections import defaultdict
from copy import deepcopy
from typing import Any, Optional

import pandas as pd

from browsergym.experiments.loop import EnvArgs

def task_metadata(benchmark_name: str):
    return task_metadata_from_csv(
        io.StringIO(pkgutil.get_data(__name__, f"{benchmark_name}.csv").decode("utf-8"))
    )

def task_metadata_from_csv(filepath):
    return pd.read_csv(filepath).fillna("")

def task_list_from_metadata(metadata: pd.DataFrame, filter: dict[str, str] = {}):
    df = metadata
    # filter the desired columns (AND filter)
    for col_name, regex in filter.items():
        col_filter = df[col_name].astype(str).str.contains(regex, regex=True)
        df = df[col_filter]
    # return only the task names
    return list(df["task_name"])

def build_full_task_dependency_graph_from_metadata(
    task_metadata: pd.DataFrame,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    if "depends_on" not in task_metadata.columns:
        raise ValueError(
            'task metadata column "depends_on" is required to compute task dependencies.'
        )

    graph_parents = {
        task_name: depends_on.split()
        for task_name, depends_on in zip(task_metadata["task_name"], task_metadata["depends_on"])
    }

    return graph_parents

def extract_graph_children(parents: dict[Any, list[Any]]):
    # start with empty children


def task_metadata(benchmark_name: str):
    return task_metadata_from_csv(
        io.StringIO(pkgutil.get_data(__name__, f"{benchmark_name}.csv").decode("utf-8"))
    )

def task_metadata_from_csv(filepath):
    return pd.read_csv(filepath).fillna("")

def task_list_from_metadata(metadata: pd.DataFrame, filter: dict[str, str] = {}):
    df = metadata
    # filter the desired columns (AND filter)
    for col_name, regex in filter.items():
        col_filter = df[col_name].astype(str).str.contains(regex, regex=True)
        df = df[col_filter]
    # return only the task names
    return list(df["task_name"])

def build_full_task_dependency_graph_from_metadata(
    task_metadata: pd.DataFrame,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    if "depends_on" not in task_metadata.columns:
        raise ValueError(
            'task metadata column "depends_on" is required to compute task dependencies.'
        )

    graph_parents = {
        task_name: depends_on.split()
        for task_name, depends_on in zip(task_metadata["task_name"], task_metadata["depends_on"])
    }

    return graph_parents

def extract_graph_children(parents: dict[Any, list[Any]]):
    # start with empty children
    children = {task_name: [] for task_name in parents.keys()}
    # build children sets iteratively
    for child, parents in parents.items():
        for parent in parents:
            children[parent].append(child)
    return children

def extract_sparse_task_dependency_graph_from_subset(
    task_subset: list[str],
    parents: dict[str, list[str]],
    children: Optional[dict[str, list[str]]] = None,
    return_children: bool = False,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:

    if children is None:


    # return only the task names
    return list(df["task_name"])

def build_full_task_dependency_graph_from_metadata(
    task_metadata: pd.DataFrame,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    if "depends_on" not in task_metadata.columns:
        raise ValueError(
            'task metadata column "depends_on" is required to compute task dependencies.'
        )

    graph_parents = {
        task_name: depends_on.split()
        for task_name, depends_on in zip(task_metadata["task_name"], task_metadata["depends_on"])
    }

    return graph_parents

def extract_graph_children(parents: dict[Any, list[Any]]):
    # start with empty children
    children = {task_name: [] for task_name in parents.keys()}
    # build children sets iteratively
    for child, parents in parents.items():
        for parent in parents:
            children[parent].append(child)
    return children

def extract_sparse_task_dependency_graph_from_subset(
    task_subset: list[str],
    parents: dict[str, list[str]],
    children: Optional[dict[str, list[str]]] = None,
    return_children: bool = False,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:

    if children is None:
        children = extract_graph_children(parents)

    # consistency check
    assert all([task in parents for task in task_subset])
    assert all([task in children for task in task_subset])

    # copy the graph
    subgraph_parents = deepcopy(parents)
    subgraph_children = deepcopy(children)


        )

    graph_parents = {
        task_name: depends_on.split()
        for task_name, depends_on in zip(task_metadata["task_name"], task_metadata["depends_on"])
    }

    return graph_parents

def extract_graph_children(parents: dict[Any, list[Any]]):
    # start with empty children
    children = {task_name: [] for task_name in parents.keys()}
    # build children sets iteratively
    for child, parents in parents.items():
        for parent in parents:
            children[parent].append(child)
    return children

def extract_sparse_task_dependency_graph_from_subset(
    task_subset: list[str],
    parents: dict[str, list[str]],
    children: Optional[dict[str, list[str]]] = None,
    return_children: bool = False,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:

    if children is None:
        children = extract_graph_children(parents)

    # consistency check
    assert all([task in parents for task in task_subset])
    assert all([task in children for task in task_subset])

    # copy the graph
    subgraph_parents = deepcopy(parents)
    subgraph_children = deepcopy(children)

    # prune the graph (node contraction)
    for task_name in parents.keys():
        # if task is not present in the target task subset, drop its node
        if task_name not in task_subset:
            # connect node's children to node's parents
            for child in subgraph_children[task_name]:
                subgraph_parents[child].remove(task_name)
                subgraph_parents[child].extend(subgraph_parents[task_name])
            # connect node's parents to node's children
            for parent in subgraph_parents[task_name]:
                subgraph_children[parent].remove(task_name)
                subgraph_children[parent].extend(subgraph_children[task_name])
            # remove node
            del subgraph_parents[task_name]
            del subgraph_children[task_name]

    # return parent and (optionally) children mappings
    if return_children:
        return subgraph_parents, subgraph_children

    return subgraph_parents

def build_env_args_dependency_graphs(
    env_args_list: list[EnvArgs],
    task_dependencies: dict[str, list[str]],
    supports_parallel_seeds: bool,
) -> list[dict[int, list[int]]]:
    """
    Returns a list of dependency graphs to be executed sequentially, typically with a full instance reset in-between.
    Ideally, a job scheduler should connect these graphs by injecting a reset task in-between each, which depends on all previous tasks being completed.
    """
    # build mapping from tasks to run subsets (task_name -> list of env_args_list indices)
    task_runs = defaultdict(list)
    for i, env_args in enumerate(env_args_list):
        task_runs[env_args.task_name].append(i)
    task_runs = dict(task_runs)

    # consistency check
    assert all([task in task_dependencies for task in task_runs.keys()])


        if task_name not in task_subset:
            # connect node's children to node's parents
            for child in subgraph_children[task_name]:
                subgraph_parents[child].remove(task_name)
                subgraph_parents[child].extend(subgraph_parents[task_name])
            # connect node's parents to node's children
            for parent in subgraph_parents[task_name]:
                subgraph_children[parent].remove(task_name)
                subgraph_children[parent].extend(subgraph_children[task_name])
            # remove node
            del subgraph_parents[task_name]
            del subgraph_children[task_name]

    # return parent and (optionally) children mappings
    if return_children:
        return subgraph_parents, subgraph_children

    return subgraph_parents

def build_env_args_dependency_graphs(
    env_args_list: list[EnvArgs],
    task_dependencies: dict[str, list[str]],
    supports_parallel_seeds: bool,
) -> list[dict[int, list[int]]]:
    """
    Returns a list of dependency graphs to be executed sequentially, typically with a full instance reset in-between.
    Ideally, a job scheduler should connect these graphs by injecting a reset task in-between each, which depends on all previous tasks being completed.
    """
    # build mapping from tasks to run subsets (task_name -> list of env_args_list indices)
    task_runs = defaultdict(list)
    for i, env_args in enumerate(env_args_list):
        task_runs[env_args.task_name].append(i)
    task_runs = dict(task_runs)

    # consistency check
    assert all([task in task_dependencies for task in task_runs.keys()])

    # divide same-task runs into distinct splits if needed
    task_runs_splits = []
    if supports_parallel_seeds:
        # if parallel task runs over seeds is supported, do not split
        task_runs_splits.append(task_runs)
    else:
        # else, split runs so that each task has only one run (seed) in each split
        while task_runs:
            # extract first task run only (one seed per task)
            split = {task_name: [runs.pop(0)] for task_name, runs in task_runs.items()}
            task_runs_splits.append(split)
            # update task list to only those with remaining runs (seeds)
            task_runs = {task_name: runs for task_name, runs in task_runs.items() if runs}

    # recover the parent and child mappings of the task dependency graph
    task_parents = task_dependencies
    task_children = extract_graph_children(task_parents)

    # for each split, build the task dependency subgraph (task_name nodes), then build the run dependency graph (env_args index nodes)
    run_parents_split = []
    for split in task_runs_splits:
        # build the task dependency graph (task_name nodes)
        split_task_names = list(split.keys())
        split_task_parents = extract_sparse_task_dependency_graph_from_subset(
            task_subset=split_task_names,
            parents=task_parents,
            children=task_children,
        )
        # then, build the run dependency graph (env_args index nodes)
        split_run_parents = defaultdict(list)
        for task_name, runs in split.items():
            parent_tasks_runs = sum([split[parent] for parent in split_task_parents[task_name]], [])
            for run in runs:
                split_run_parents[run].extend(parent_tasks_runs)
        split_run_parents = dict(split_run_parents)
        run_parents_split.append(split_run_parents)

    return run_parents_split


import importlib.resources
import json

import numpy as np

from browsergym.experiments.benchmark import task_metadata

# for posterity

def print_metadata_workarena():
    from browsergym.workarena import (
        AGENT_CURRICULUM_L2,
        AGENT_CURRICULUM_L3,
        TASK_CATEGORY_MAP,
    )

    metadata = [("task_name", "level", "category")]

    for task_name, category in TASK_CATEGORY_MAP.items():
        metadata.append((task_name, "l1", category))

    for category, items in AGENT_CURRICULUM_L2.items():
        for task_set in items["buckets"]:
            for task in task_set:
                metadata.append((task.get_task_id(), "l2", category))

    for category, items in AGENT_CURRICULUM_L3.items():
        for task_set in items["buckets"]:
            for task in task_set:
                metadata.append((task.get_task_id(), "l3", category))

    print("\n".join([",".join(x) for x in metadata]))

def print_metadata_webarena():

    import webarena

    metadata = [
        (
            "task_name",
            "requires_reset",
            "sites",
            "eval_types",
        )
    ]
    all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text()
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"webarena.{task['task_id']}",
                str(task["require_reset"] == True),
                " ".join(task["sites"]),
                " ".join(task["eval"]["eval_types"]),
            )
        )

    print("\n".join([",".join(x) for x in metadata]))

def print_metadata_visualwebarena():
    import visualwebarena

    metadata = [
        (
            "task_name",
            "requires_reset",
            "sites",
            "reasoning_difficulty",
            "visual_difficulty",
            "overall_difficulty",
            "eval_types",
        )
    ]

    all_configs_str = (
        importlib.resources.files(visualwebarena).joinpath("test_raw.json").read_text()
    )
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"visualwebarena.{task['task_id']}",
                str(task["require_reset"] == True),
                " ".join(task["sites"]),
                task["reasoning_difficulty"],
                task["visual_difficulty"],
                task["overall_difficulty"],
                " ".join(task["eval"]["eval_types"]),
            )
        )

    print("\n".join([",".join(x) for x in metadata]))

def print_miniwob_train_test_splits():
    metadata = task_metadata("miniwob")

    groups = metadata["similarity_group"]
    group_counts = groups.value_counts(sort=False)

    group_counts = dict({group: count for group, count in zip(group_counts.index, group_counts)})

    free_groups = set(group_counts.keys())
    train_groups = set()
    test_groups = set()
    rng = np.random.RandomState(1337)

    # slack for train / test size equality
    slack = sum(group_counts.values()) % 2

    def move_random_group(from_groups: set, to_groups: set):
        # pick uniformly among tasks (weighted sampling among groups)
        probs = np.asarray([float(group_counts[group]) for group in from_groups])
        probs = probs / probs.sum()
        # sample a group
        group = rng.choice(list(from_groups), size=1, p=probs)[0]
        # move between sets
        to_groups.add(group)
        from_groups.remove(group)
        # return group for information
        return group

    done = False
    while not done:
        n_train = sum([group_counts[group] for group in train_groups])
        n_test = sum([group_counts[group] for group in test_groups])

        print(f"train/test split: {n_train} <> {n_test}")

        # growing phase
        if free_groups:
            if n_train < n_test:
                group = move_random_group(from_groups=free_groups, to_groups=train_groups)
                print(f"adding {group} to train")
            else:
                group = move_random_group(from_groups=free_groups, to_groups=test_groups)
                print(f"adding {group} to test")

        # group switching phase
        elif n_train < n_test - slack:
            group = move_random_group(from_groups=test_groups, to_groups=train_groups)
            print(f"switching {group} from test to train")
        elif n_test < n_train - slack:
            group = move_random_group(from_groups=train_groups, to_groups=test_groups)
            print(f"switching {group} from train to test")

        # done (equilibrium)
        else:
            print("equilibrium")
            done = True

        print()

    metadata["browsergym_split"] = metadata["similarity_group"].apply(
        lambda group: "train" if group in train_groups else "test" if group in test_groups else ""
    )

    print(metadata.to_csv(index=False))


import importlib.resources
import json

import numpy as np

from browsergym.experiments.benchmark import task_metadata

# for posterity

def print_metadata_workarena():
    from browsergym.workarena import (
        AGENT_CURRICULUM_L2,
        AGENT_CURRICULUM_L3,
        TASK_CATEGORY_MAP,
    )

    metadata = [("task_name", "level", "category")]

    for task_name, category in TASK_CATEGORY_MAP.items():
        metadata.append((task_name, "l1", category))

    for category, items in AGENT_CURRICULUM_L2.items():
        for task_set in items["buckets"]:
            for task in task_set:
                metadata.append((task.get_task_id(), "l2", category))

    for category, items in AGENT_CURRICULUM_L3.items():
        for task_set in items["buckets"]:
            for task in task_set:
                metadata.append((task.get_task_id(), "l3", category))

    print("\n".join([",".join(x) for x in metadata]))

def print_metadata_webarena():

    import webarena

    metadata = [
        (
            "task_name",
            "requires_reset",
            "sites",
            "eval_types",
        )
    ]
    all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text()
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"webarena.{task['task_id']}",


    )

    metadata = [("task_name", "level", "category")]

    for task_name, category in TASK_CATEGORY_MAP.items():
        metadata.append((task_name, "l1", category))

    for category, items in AGENT_CURRICULUM_L2.items():
        for task_set in items["buckets"]:
            for task in task_set:
                metadata.append((task.get_task_id(), "l2", category))

    for category, items in AGENT_CURRICULUM_L3.items():
        for task_set in items["buckets"]:
            for task in task_set:
                metadata.append((task.get_task_id(), "l3", category))

    print("\n".join([",".join(x) for x in metadata]))

def print_metadata_webarena():

    import webarena

    metadata = [
        (
            "task_name",
            "requires_reset",
            "sites",
            "eval_types",
        )
    ]
    all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text()
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"webarena.{task['task_id']}",
                str(task["require_reset"] == True),
                " ".join(task["sites"]),
                " ".join(task["eval"]["eval_types"]),
            )
        )

    print("\n".join([",".join(x) for x in metadata]))

def print_metadata_visualwebarena():
    import visualwebarena

    metadata = [
        (
            "task_name",
            "requires_reset",
            "sites",
            "reasoning_difficulty",
            "visual_difficulty",
            "overall_difficulty",
            "eval_types",
        )
    ]

    all_configs_str = (
        importlib.resources.files(visualwebarena).joinpath("test_raw.json").read_text()
    )


            "requires_reset",
            "sites",
            "eval_types",
        )
    ]
    all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text()
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"webarena.{task['task_id']}",
                str(task["require_reset"] == True),
                " ".join(task["sites"]),
                " ".join(task["eval"]["eval_types"]),
            )
        )

    print("\n".join([",".join(x) for x in metadata]))

def print_metadata_visualwebarena():
    import visualwebarena

    metadata = [
        (
            "task_name",
            "requires_reset",
            "sites",
            "reasoning_difficulty",
            "visual_difficulty",
            "overall_difficulty",
            "eval_types",
        )
    ]

    all_configs_str = (
        importlib.resources.files(visualwebarena).joinpath("test_raw.json").read_text()
    )
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"visualwebarena.{task['task_id']}",
                str(task["require_reset"] == True),
                " ".join(task["sites"]),
                task["reasoning_difficulty"],
                task["visual_difficulty"],
                task["overall_difficulty"],
                " ".join(task["eval"]["eval_types"]),
            )
        )

    print("\n".join([",".join(x) for x in metadata]))

def print_miniwob_train_test_splits():
    metadata = task_metadata("miniwob")

    groups = metadata["similarity_group"]
    group_counts = groups.value_counts(sort=False)

    group_counts = dict({group: count for group, count in zip(group_counts.index, group_counts)})

    free_groups = set(group_counts.keys())
    train_groups = set()
    test_groups = set()
    rng = np.random.RandomState(1337)

    # slack for train / test size equality
    slack = sum(group_counts.values()) % 2

    def move_random_group(from_groups: set, to_groups: set):
        # pick uniformly among tasks (weighted sampling among groups)


    all_configs_str = (
        importlib.resources.files(visualwebarena).joinpath("test_raw.json").read_text()
    )
    all_configs = json.loads(all_configs_str)
    for task in all_configs:
        metadata.append(
            (
                f"visualwebarena.{task['task_id']}",
                str(task["require_reset"] == True),
                " ".join(task["sites"]),
                task["reasoning_difficulty"],
                task["visual_difficulty"],
                task["overall_difficulty"],
                " ".join(task["eval"]["eval_types"]),
            )
        )

    print("\n".join([",".join(x) for x in metadata]))

def print_miniwob_train_test_splits():
    metadata = task_metadata("miniwob")

    groups = metadata["similarity_group"]
    group_counts = groups.value_counts(sort=False)

    group_counts = dict({group: count for group, count in zip(group_counts.index, group_counts)})

    free_groups = set(group_counts.keys())
    train_groups = set()
    test_groups = set()
    rng = np.random.RandomState(1337)

    # slack for train / test size equality
    slack = sum(group_counts.values()) % 2

    def move_random_group(from_groups: set, to_groups: set):
        # pick uniformly among tasks (weighted sampling among groups)
        probs = np.asarray([float(group_counts[group]) for group in from_groups])
        probs = probs / probs.sum()
        # sample a group
        group = rng.choice(list(from_groups), size=1, p=probs)[0]
        # move between sets
        to_groups.add(group)
        from_groups.remove(group)
        # return group for information
        return group

    done = False
    while not done:
        n_train = sum([group_counts[group] for group in train_groups])
        n_test = sum([group_counts[group] for group in test_groups])

        print(f"train/test split: {n_train} <> {n_test}")

        # growing phase
        if free_groups:
            if n_train < n_test:
                group = move_random_group(from_groups=free_groups, to_groups=train_groups)
                print(f"adding {group} to train")
            else:
                group = move_random_group(from_groups=free_groups, to_groups=test_groups)
                print(f"adding {group} to test")

        # group switching phase
        elif n_train < n_test - slack:
            group = move_random_group(from_groups=test_groups, to_groups=train_groups)
            print(f"switching {group} from test to train")
        elif n_test < n_train - slack:
            group = move_random_group(from_groups=train_groups, to_groups=test_groups)
            print(f"switching {group} from train to test")

        # done (equilibrium)
        else:
            print("equilibrium")
            done = True

        print()

    metadata["browsergym_split"] = metadata["similarity_group"].apply(
        lambda group: "train" if group in train_groups else "test" if group in test_groups else ""
    )

    print(metadata.to_csv(index=False))


    print("\n".join([",".join(x) for x in metadata]))

def print_miniwob_train_test_splits():
    metadata = task_metadata("miniwob")

    groups = metadata["similarity_group"]
    group_counts = groups.value_counts(sort=False)

    group_counts = dict({group: count for group, count in zip(group_counts.index, group_counts)})

    free_groups = set(group_counts.keys())
    train_groups = set()
    test_groups = set()
    rng = np.random.RandomState(1337)

    # slack for train / test size equality
    slack = sum(group_counts.values()) % 2

    def move_random_group(from_groups: set, to_groups: set):
        # pick uniformly among tasks (weighted sampling among groups)
        probs = np.asarray([float(group_counts[group]) for group in from_groups])
        probs = probs / probs.sum()
        # sample a group
        group = rng.choice(list(from_groups), size=1, p=probs)[0]
        # move between sets
        to_groups.add(group)
        from_groups.remove(group)
        # return group for information
        return group

    done = False
    while not done:
        n_train = sum([group_counts[group] for group in train_groups])
        n_test = sum([group_counts[group] for group in test_groups])

        print(f"train/test split: {n_train} <> {n_test}")

        # growing phase
        if free_groups:
            if n_train < n_test:
                group = move_random_group(from_groups=free_groups, to_groups=train_groups)
                print(f"adding {group} to train")
            else:
                group = move_random_group(from_groups=free_groups, to_groups=test_groups)
                print(f"adding {group} to test")

        # group switching phase
        elif n_train < n_test - slack:
            group = move_random_group(from_groups=test_groups, to_groups=train_groups)


import json
import logging
import os
import pathlib
import time

logger = logging.getLogger(__name__)

def add_prediction_to_jsonl(
    file_path: str, task_id: str, prediction: object, override_if_exists: bool
) -> None:
    """
    Multiprocessing-safe file write.
    """
    lock_file_path = pathlib.Path(file_path).with_suffix(".lock")
    lock_max_wait = 10  # 10 seconds

    # Acquire lock (atomic file creation)
    start_time = time.time()
    while True:
        try:
            fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
            with os.fdopen(fd, "w") as f:
                f.write("lock")
            break
        except FileExistsError:
            # give up if max wait time reached
            seconds_waited = time.time() - start_time
            if seconds_waited >= lock_max_wait:
                raise RuntimeError(
                    f"Lock file could not be acquired after {seconds_waited} seconds ({lock_file_path})"
                )
            # wait for lock release
            logger.info(f"Waiting for lock file to be released: {lock_file_path}")
            time.sleep(1)  # 1 sec

    logger.info(f"Lock file acquired: {lock_file_path}")

    # Check if the file exists, if not, create it
    if not os.path.exists(file_path):
        with open(file_path, "w") as f:
            pass  # Create an empty file

    # Load existing data, if any
    data = []
    if os.path.exists(file_path):
        with open(file_path, "r") as f:
            data.extend([json.loads(line) for line in f if line.strip()])  # Skip empty lines

    # Check if task_id already exists
    existing_record = next((entry for entry in data if entry["id"] == task_id), None)

    # Add or update the record
    if not existing_record:
        # Add new record
        data.append({"id": task_id, "answer": prediction})
    elif override_if_exists:
        # Update existing record
        existing_record["answer"] = prediction
    else:
        raise ValueError(
            f"Prediction for task ID {repr(task_id)} already exists in file {file_path}."
        )

    # Write data back to the file
    with open(file_path, "w") as f:
        for entry in data:
            f.write(json.dumps(entry) + "\n")

    # Release lock (remove file)
    os.remove(lock_file_path)
    logger.info(f"Lock file released: {lock_file_path}")


import logging
import os
from typing import Dict, Tuple

from datasets import load_dataset
from playwright.sync_api import Page

from browsergym.core.task import AbstractBrowserTask

from .evaluation.evaluator import question_scorer
from .utils import add_prediction_to_jsonl

logger = logging.getLogger(__name__)

_DEFAULT_OUTPUT_FILE = None

def set_default_output_file(output_file: str):
    global _DEFAULT_OUTPUT_FILE
    _DEFAULT_OUTPUT_FILE = output_file

def get_default_output_file():
    return _DEFAULT_OUTPUT_FILE

# Load dataset

DATA_DATASET = "AssistantBench/AssistantBench"
all_tasks = load_dataset(DATA_DATASET, trust_remote_code=True)

# Extract answers and tasks for validation and test splits
def extract_data(split_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {
            f"{split_name}.{i}": row["answer"] if row["answer"] is not None else ""
            for i, row in enumerate(all_tasks[split_name])
        },
        {f"{split_name}.{i}": row["task"] for i, row in enumerate(all_tasks[split_name])},
        {f"{split_name}.{i}": row["id"] for i, row in enumerate(all_tasks[split_name])},
    )

# Implementation data for testing
def get_implementation_testing_data() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {"imp.0": "20"},
        {
            "imp.0": "What is the weather in Paris yesterday in Celsius? Answer with the number only."
        },
        {"imp.0": "test_imp_id_0"},
    )

# Combine dev, test, and implementation-specific testing splits
gold_answers_dev, tasks_dev, ids_dev = extract_data("validation")
gold_answers_test, tasks_test, ids_test = extract_data("test")
gold_answers_impl_testing, tasks_test_impl_testing, ids_imp_testing = (
    get_implementation_testing_data()
)
gold_answers = {**gold_answers_dev, **gold_answers_test, **gold_answers_impl_testing}
tasks = {**tasks_dev, **tasks_test, **tasks_test_impl_testing}
ids = {**ids_dev, **ids_test, **ids_imp_testing}

class AssistantBenchTask(AbstractBrowserTask):

    @classmethod
    def get_task_id(cls) -> str:
        """
        Generic class for several task ids, this way of obtaining the task id is not compatible for now.
        """
        raise NotImplementedError

    def __init__(
        self, seed: int, task_id: str, output_file: str = None, save_predictions: bool = False
    ) -> None:
        """
        Args:
            seed (int): Random seed for task initialization.
            task_id (str): Unique identifier for the task (for the BrowserGym environment).
            output_file (str, optional): Path to the output file for saving results, needed for test set.
            save_predictions (bool, optional): Save predictions to the output file (yes/no).
        """
        super().__init__(seed)
        self.locale = "en-US"
        self.timezone_id = "America/New_York"

        self.task_id = task_id
        self.start_url = "https://google.com"
        self.goal = tasks[str(self.task_id)]
        self.gold = gold_answers[str(self.task_id)]
        self.ab_task_id = ids[self.task_id]
        self.save_predictions = save_predictions

        self.output_file = output_file

        # set output_file using the global default value, if not provided in constructor
        if not self.output_file:
            self.output_file = get_default_output_file()
        # use env variable in last resort
        if not self.output_file:
            self.output_file = os.getenv("ASSISTANTBENCH_OUTPUT_FILE", None)

        if self.save_predictions and self.output_file:
            logger.info(f"Task prediction will be written to output file {self.output_file}")

    def setup(self, page: Page) -> Tuple[str, dict]:
        logger.info(f"Navigating to start url: {self.start_url}")
        page.goto(self.start_url, timeout=10000)
        if self.save_predictions and self.output_file:
            # create an empty task entry in the output file (will raise an Exception if the entry is already there)
            add_prediction_to_jsonl(
                file_path=self.output_file,
                task_id=self.ab_task_id,
                prediction="",
                override_if_exists=False,
            )
        return self.goal, {}

    def teardown(self) -> None:
        pass

    def validate(self, page: Page, chat_messages: list[dict]) -> Tuple[float, bool, str, dict]:
        accuracy, done, msg, info = 0.0, False, "", {}

        # eval when the agent returns a response
        if chat_messages and chat_messages[-1]["role"] == "assistant":
            done = True
            prediction = chat_messages[-1]["message"]
            if self.save_predictions and self.output_file:
                # update the task entry in the output file
                add_prediction_to_jsonl(
                    file_path=self.output_file,
                    task_id=self.ab_task_id,
                    prediction=prediction,
                    override_if_exists=True,
                )
            accuracy, has_ans = question_scorer(prediction, self.gold)

        return accuracy, done, msg, info


import logging
import os
from typing import Dict, Tuple

from datasets import load_dataset
from playwright.sync_api import Page

from browsergym.core.task import AbstractBrowserTask

from .evaluation.evaluator import question_scorer
from .utils import add_prediction_to_jsonl

logger = logging.getLogger(__name__)

_DEFAULT_OUTPUT_FILE = None

def set_default_output_file(output_file: str):
    global _DEFAULT_OUTPUT_FILE
    _DEFAULT_OUTPUT_FILE = output_file

def get_default_output_file():
    return _DEFAULT_OUTPUT_FILE

# Load dataset

DATA_DATASET = "AssistantBench/AssistantBench"
all_tasks = load_dataset(DATA_DATASET, trust_remote_code=True)

# Extract answers and tasks for validation and test splits
def extract_data(split_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {
            f"{split_name}.{i}": row["answer"] if row["answer"] is not None else ""
            for i, row in enumerate(all_tasks[split_name])
        },
        {f"{split_name}.{i}": row["task"] for i, row in enumerate(all_tasks[split_name])},


from typing import Dict, Tuple

from datasets import load_dataset
from playwright.sync_api import Page

from browsergym.core.task import AbstractBrowserTask

from .evaluation.evaluator import question_scorer
from .utils import add_prediction_to_jsonl

logger = logging.getLogger(__name__)

_DEFAULT_OUTPUT_FILE = None

def set_default_output_file(output_file: str):
    global _DEFAULT_OUTPUT_FILE
    _DEFAULT_OUTPUT_FILE = output_file

def get_default_output_file():
    return _DEFAULT_OUTPUT_FILE

# Load dataset

DATA_DATASET = "AssistantBench/AssistantBench"
all_tasks = load_dataset(DATA_DATASET, trust_remote_code=True)

# Extract answers and tasks for validation and test splits
def extract_data(split_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {
            f"{split_name}.{i}": row["answer"] if row["answer"] is not None else ""
            for i, row in enumerate(all_tasks[split_name])
        },
        {f"{split_name}.{i}": row["task"] for i, row in enumerate(all_tasks[split_name])},
        {f"{split_name}.{i}": row["id"] for i, row in enumerate(all_tasks[split_name])},
    )


_DEFAULT_OUTPUT_FILE = None

def set_default_output_file(output_file: str):
    global _DEFAULT_OUTPUT_FILE
    _DEFAULT_OUTPUT_FILE = output_file

def get_default_output_file():
    return _DEFAULT_OUTPUT_FILE

# Load dataset

DATA_DATASET = "AssistantBench/AssistantBench"
all_tasks = load_dataset(DATA_DATASET, trust_remote_code=True)

# Extract answers and tasks for validation and test splits
def extract_data(split_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {
            f"{split_name}.{i}": row["answer"] if row["answer"] is not None else ""
            for i, row in enumerate(all_tasks[split_name])
        },
        {f"{split_name}.{i}": row["task"] for i, row in enumerate(all_tasks[split_name])},
        {f"{split_name}.{i}": row["id"] for i, row in enumerate(all_tasks[split_name])},
    )

# Implementation data for testing
def get_implementation_testing_data() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {"imp.0": "20"},
        {
            "imp.0": "What is the weather in Paris yesterday in Celsius? Answer with the number only."
        },
        {"imp.0": "test_imp_id_0"},
    )

# Combine dev, test, and implementation-specific testing splits
gold_answers_dev, tasks_dev, ids_dev = extract_data("validation")
gold_answers_test, tasks_test, ids_test = extract_data("test")
gold_answers_impl_testing, tasks_test_impl_testing, ids_imp_testing = (
    get_implementation_testing_data()
)
gold_answers = {**gold_answers_dev, **gold_answers_test, **gold_answers_impl_testing}


# Load dataset

DATA_DATASET = "AssistantBench/AssistantBench"
all_tasks = load_dataset(DATA_DATASET, trust_remote_code=True)

# Extract answers and tasks for validation and test splits
def extract_data(split_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {
            f"{split_name}.{i}": row["answer"] if row["answer"] is not None else ""
            for i, row in enumerate(all_tasks[split_name])
        },
        {f"{split_name}.{i}": row["task"] for i, row in enumerate(all_tasks[split_name])},
        {f"{split_name}.{i}": row["id"] for i, row in enumerate(all_tasks[split_name])},
    )

# Implementation data for testing
def get_implementation_testing_data() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
    return (
        {"imp.0": "20"},
        {
            "imp.0": "What is the weather in Paris yesterday in Celsius? Answer with the number only."
        },
        {"imp.0": "test_imp_id_0"},
    )

# Combine dev, test, and implementation-specific testing splits
gold_answers_dev, tasks_dev, ids_dev = extract_data("validation")
gold_answers_test, tasks_test, ids_test = extract_data("test")
gold_answers_impl_testing, tasks_test_impl_testing, ids_imp_testing = (
    get_implementation_testing_data()
)
gold_answers = {**gold_answers_dev, **gold_answers_test, **gold_answers_impl_testing}
tasks = {**tasks_dev, **tasks_test, **tasks_test_impl_testing}
ids = {**ids_dev, **ids_test, **ids_imp_testing}

class AssistantBenchTask(AbstractBrowserTask):

    @classmethod
    def get_task_id(cls) -> str:
        """
        Generic class for several task ids, this way of obtaining the task id is not compatible for now.
        """


AssistantBenchTask

    return (
        {"imp.0": "20"},
        {
            "imp.0": "What is the weather in Paris yesterday in Celsius? Answer with the number only."
        },
        {"imp.0": "test_imp_id_0"},
    )

# Combine dev, test, and implementation-specific testing splits
gold_answers_dev, tasks_dev, ids_dev = extract_data("validation")
gold_answers_test, tasks_test, ids_test = extract_data("test")
gold_answers_impl_testing, tasks_test_impl_testing, ids_imp_testing = (
    get_implementation_testing_data()
)
gold_answers = {**gold_answers_dev, **gold_answers_test, **gold_answers_impl_testing}
tasks = {**tasks_dev, **tasks_test, **tasks_test_impl_testing}
ids = {**ids_dev, **ids_test, **ids_imp_testing}

class AssistantBenchTask(AbstractBrowserTask):

    @classmethod
    def get_task_id(cls) -> str:
        """
        Generic class for several task ids, this way of obtaining the task id is not compatible for now.
        """
        raise NotImplementedError

    def __init__(
        self, seed: int, task_id: str, output_file: str = None, save_predictions: bool = False
    ) -> None:
        """
        Args:
            seed (int): Random seed for task initialization.
            task_id (str): Unique identifier for the task (for the BrowserGym environment).
            output_file (str, optional): Path to the output file for saving results, needed for test set.
            save_predictions (bool, optional): Save predictions to the output file (yes/no).
        """
        super().__init__(seed)
        self.locale = "en-US"
        self.timezone_id = "America/New_York"

        self.task_id = task_id
        self.start_url = "https://google.com"
        self.goal = tasks[str(self.task_id)]
        self.gold = gold_answers[str(self.task_id)]
        self.ab_task_id = ids[self.task_id]
        self.save_predictions = save_predictions

        self.output_file = output_file

        # set output_file using the global default value, if not provided in constructor
        if not self.output_file:
            self.output_file = get_default_output_file()
        # use env variable in last resort
        if not self.output_file:
            self.output_file = os.getenv("ASSISTANTBENCH_OUTPUT_FILE", None)

        if self.save_predictions and self.output_file:
            logger.info(f"Task prediction will be written to output file {self.output_file}")

    def setup(self, page: Page) -> Tuple[str, dict]:
        logger.info(f"Navigating to start url: {self.start_url}")
        page.goto(self.start_url, timeout=10000)
        if self.save_predictions and self.output_file:
            # create an empty task entry in the output file (will raise an Exception if the entry is already there)
            add_prediction_to_jsonl(
                file_path=self.output_file,
                task_id=self.ab_task_id,
                prediction="",
                override_if_exists=False,
            )
        return self.goal, {}

    def teardown(self) -> None:
        pass

    def validate(self, page: Page, chat_messages: list[dict]) -> Tuple[float, bool, str, dict]:
        accuracy, done, msg, info = 0.0, False, "", {}

        # eval when the agent returns a response
        if chat_messages and chat_messages[-1]["role"] == "assistant":
            done = True
            prediction = chat_messages[-1]["message"]
            if self.save_predictions and self.output_file:
                # update the task entry in the output file
                add_prediction_to_jsonl(
                    file_path=self.output_file,
                    task_id=self.ab_task_id,
                    prediction=prediction,
                    override_if_exists=True,
                )
            accuracy, has_ans = question_scorer(prediction, self.gold)

        return accuracy, done, msg, info


            "imp.0": "What is the weather in Paris yesterday in Celsius? Answer with the number only."
        },
        {"imp.0": "test_imp_id_0"},
    )

# Combine dev, test, and implementation-specific testing splits
gold_answers_dev, tasks_dev, ids_dev = extract_data("validation")
gold_answers_test, tasks_test, ids_test = extract_data("test")
gold_answers_impl_testing, tasks_test_impl_testing, ids_imp_testing = (
    get_implementation_testing_data()
)
gold_answers = {**gold_answers_dev, **gold_answers_test, **gold_answers_impl_testing}
tasks = {**tasks_dev, **tasks_test, **tasks_test_impl_testing}
ids = {**ids_dev, **ids_test, **ids_imp_testing}

class AssistantBenchTask(AbstractBrowserTask):

    @classmethod
    def get_task_id(cls) -> str:
        """
        Generic class for several task ids, this way of obtaining the task id is not compatible for now.
        """
        raise NotImplementedError

    def __init__(
        self, seed: int, task_id: str, output_file: str = None, save_predictions: bool = False
    ) -> None:
        """
        Args:
            seed (int): Random seed for task initialization.
            task_id (str): Unique identifier for the task (for the BrowserGym environment).
            output_file (str, optional): Path to the output file for saving results, needed for test set.
            save_predictions (bool, optional): Save predictions to the output file (yes/no).
        """
        super().__init__(seed)
        self.locale = "en-US"
        self.timezone_id = "America/New_York"

        self.task_id = task_id
        self.start_url = "https://google.com"
        self.goal = tasks[str(self.task_id)]
        self.gold = gold_answers[str(self.task_id)]
        self.ab_task_id = ids[self.task_id]


# Combine dev, test, and implementation-specific testing splits
gold_answers_dev, tasks_dev, ids_dev = extract_data("validation")
gold_answers_test, tasks_test, ids_test = extract_data("test")
gold_answers_impl_testing, tasks_test_impl_testing, ids_imp_testing = (
    get_implementation_testing_data()
)
gold_answers = {**gold_answers_dev, **gold_answers_test, **gold_answers_impl_testing}
tasks = {**tasks_dev, **tasks_test, **tasks_test_impl_testing}
ids = {**ids_dev, **ids_test, **ids_imp_testing}

class AssistantBenchTask(AbstractBrowserTask):

    @classmethod
    def get_task_id(cls) -> str:
        """
        Generic class for several task ids, this way of obtaining the task id is not compatible for now.
        """
        raise NotImplementedError

    def __init__(
        self, seed: int, task_id: str, output_file: str = None, save_predictions: bool = False
    ) -> None:
        """
        Args:
            seed (int): Random seed for task initialization.
            task_id (str): Unique identifier for the task (for the BrowserGym environment).
            output_file (str, optional): Path to the output file for saving results, needed for test set.
            save_predictions (bool, optional): Save predictions to the output file (yes/no).
        """
        super().__init__(seed)
        self.locale = "en-US"
        self.timezone_id = "America/New_York"

        self.task_id = task_id
        self.start_url = "https://google.com"
        self.goal = tasks[str(self.task_id)]
        self.gold = gold_answers[str(self.task_id)]
        self.ab_task_id = ids[self.task_id]
        self.save_predictions = save_predictions

        self.output_file = output_file

        # set output_file using the global default value, if not provided in constructor
        if not self.output_file:
            self.output_file = get_default_output_file()
        # use env variable in last resort
        if not self.output_file:
            self.output_file = os.getenv("ASSISTANTBENCH_OUTPUT_FILE", None)

        if self.save_predictions and self.output_file:
            logger.info(f"Task prediction will be written to output file {self.output_file}")

    def setup(self, page: Page) -> Tuple[str, dict]:
        logger.info(f"Navigating to start url: {self.start_url}")
        page.goto(self.start_url, timeout=10000)
        if self.save_predictions and self.output_file:
            # create an empty task entry in the output file (will raise an Exception if the entry is already there)
            add_prediction_to_jsonl(
                file_path=self.output_file,
                task_id=self.ab_task_id,
                prediction="",
                override_if_exists=False,
            )
        return self.goal, {}

    def teardown(self) -> None:
        pass

    def validate(self, page: Page, chat_messages: list[dict]) -> Tuple[float, bool, str, dict]:
        accuracy, done, msg, info = 0.0, False, "", {}


setup

        self.task_id = task_id
        self.start_url = "https://google.com"
        self.goal = tasks[str(self.task_id)]
        self.gold = gold_answers[str(self.task_id)]
        self.ab_task_id = ids[self.task_id]
        self.save_predictions = save_predictions

        self.output_file = output_file

        # set output_file using the global default value, if not provided in constructor
        if not self.output_file:
            self.output_file = get_default_output_file()
        # use env variable in last resort
        if not self.output_file:
            self.output_file = os.getenv("ASSISTANTBENCH_OUTPUT_FILE", None)

        if self.save_predictions and self.output_file:
            logger.info(f"Task prediction will be written to output file {self.output_file}")

    def setup(self, page: Page) -> Tuple[str, dict]:
        logger.info(f"Navigating to start url: {self.start_url}")
        page.goto(self.start_url, timeout=10000)
        if self.save_predictions and self.output_file:
            # create an empty task entry in the output file (will raise an Exception if the entry is already there)
            add_prediction_to_jsonl(
                file_path=self.output_file,
                task_id=self.ab_task_id,
                prediction="",
                override_if_exists=False,
            )
        return self.goal, {}

    def teardown(self) -> None:
        pass

    def validate(self, page: Page, chat_messages: list[dict]) -> Tuple[float, bool, str, dict]:
        accuracy, done, msg, info = 0.0, False, "", {}

        # eval when the agent returns a response
        if chat_messages and chat_messages[-1]["role"] == "assistant":
            done = True
            prediction = chat_messages[-1]["message"]
            if self.save_predictions and self.output_file:
                # update the task entry in the output file
                add_prediction_to_jsonl(
                    file_path=self.output_file,
                    task_id=self.ab_task_id,
                    prediction=prediction,
                    override_if_exists=True,
                )
            accuracy, has_ans = question_scorer(prediction, self.gold)


teardown

        # use env variable in last resort
        if not self.output_file:
            self.output_file = os.getenv("ASSISTANTBENCH_OUTPUT_FILE", None)

        if self.save_predictions and self.output_file:
            logger.info(f"Task prediction will be written to output file {self.output_file}")

    def setup(self, page: Page) -> Tuple[str, dict]:
        logger.info(f"Navigating to start url: {self.start_url}")
        page.goto(self.start_url, timeout=10000)
        if self.save_predictions and self.output_file:
            # create an empty task entry in the output file (will raise an Exception if the entry is already there)
            add_prediction_to_jsonl(
                file_path=self.output_file,
                task_id=self.ab_task_id,
                prediction="",
                override_if_exists=False,
            )
        return self.goal, {}

    def teardown(self) -> None:
        pass

    def validate(self, page: Page, chat_messages: list[dict]) -> Tuple[float, bool, str, dict]:
        accuracy, done, msg, info = 0.0, False, "", {}

        # eval when the agent returns a response
        if chat_messages and chat_messages[-1]["role"] == "assistant":
            done = True
            prediction = chat_messages[-1]["message"]
            if self.save_predictions and self.output_file:
                # update the task entry in the output file
                add_prediction_to_jsonl(
                    file_path=self.output_file,
                    task_id=self.ab_task_id,
                    prediction=prediction,
                    override_if_exists=True,
                )
            accuracy, has_ans = question_scorer(prediction, self.gold)

        return accuracy, done, msg, info