BrowserGym/tests/core/test_actions_highlevel.py

import ast
import os
import pathlib
import platform
import re

import bs4
import gymnasium as gym
import pytest
from pyparsing.exceptions import ParseException

# register openended gym environments
import browsergym.core
from browsergym.core.action.highlevel import HighLevelActionSet
from browsergym.core.action.parsers import NamedArgument, highlevel_action_parser
from browsergym.core.constants import BROWSERGYM_ID_ATTRIBUTE as BID_ATTR
from browsergym.utils.obs import flatten_dom_to_str

_IS_MAC_OS = platform.system() == "Darwin"

__SLOW_MO = 1000 if "DISPLAY_BROWSER" in os.environ else None
__HEADLESS = False if "DISPLAY_BROWSER" in os.environ else True
__TIMEOUT = 500

__DATA_DIR = pathlib.Path(__file__).resolve().parent / "data"

TEXTBOX_URL = f"file://{__DATA_DIR}/textbox.html"
EXAMPLE_URL = f"file://{__DATA_DIR}/example.html"
HOVER_URL = f"file://{__DATA_DIR}/hover.html"
INEXISTANT_FILE_URL = f"file://{__DATA_DIR}/no_file_here.html"
LONG_PAGE_URL = f"file://{__DATA_DIR}/long_page.html"
TEXT_INPUT_URL = f"file://{__DATA_DIR}/input_type/text_input.html"
URL_INPUT_URL = f"file://{__DATA_DIR}/input_type/url_input.html"
CHECKBOX_URL = f"file://{__DATA_DIR}/input_type/checkbox_input.html"
MULTI_IFRAME_URL = f"file://{__DATA_DIR}/basic_iframe_site/basic_iframe_2.html"
OBSTRUCTED_CHECKBOX_URL = f"file://{__DATA_DIR}/obstructed_checkbox_page.html"
LOTS_OF_IFRAMES_URL = f"file://{__DATA_DIR}/lots_of_iframes.html"


def test_action_parser():
    parser = highlevel_action_parser

    with pytest.raises(ParseException):
        function_calls = parser.parse_string("", parseAll=True)
        assert not function_calls

    function_calls = parser.parse_string("a()", parseAll=True)
    assert len(function_calls) == 1

    function_calls = parser.parse_string("  a ( ) \n\n\t", parseAll=True)
    assert len(function_calls) == 1

    function_calls = parser.parse_string("  a ( ) b() \n \tc()", parseAll=True)
    assert [function_name for function_name, _ in function_calls] == ["a", "b", "c"]

    function_calls = parser.parse_string('a(12, 12.2, "text", (1, 2, 3), ["a", 23])', parseAll=True)
    _, function_args = function_calls[0]
    assert function_args == [12, 12.2, "text", (1, 2, 3), ["a", 23]]

    function_calls = parser.parse_string('a(x=12, y = 12.2, other = "text")', parseAll=True)
    _, function_args = function_calls[0]
    assert function_args == [
        NamedArgument(name="x", value=12),
        NamedArgument(name="y", value=12.2),
        NamedArgument(name="other", value="text"),
    ]

    function_calls = parser.parse_string('a(12, y = 12.2, other = "text")', parseAll=True)
    _, function_args = function_calls[0]
    assert function_args == [
        12,
        NamedArgument(name="y", value=12.2),
        NamedArgument(name="other", value="text"),
    ]

    with pytest.raises(ParseException):
        function_calls = parser.parse_string('a(x = 12, 12.2, other = "text")', parseAll=True)

    with pytest.raises(ParseException):
        function_calls = parser.parse_string('a(12, 12.2, 1 = "text")', parseAll=True)

    with pytest.raises(ParseException):
        function_calls = parser.parse_string("a(1-)", parseAll=True)

    with pytest.raises(ParseException):
        function_calls = parser.parse_string("a(1/2)", parseAll=True)

    function_calls = parser.parse_string('a("""\nsome\ntext\\"\\"""")', parseAll=True)
    _, function_args = function_calls[0]
    assert function_args == ['\nsome\ntext""']

    function_calls = parser.parse_string("a('\"some\\ntext\"')", parseAll=True)
    _, function_args = function_calls[0]
    assert function_args == ['"some\ntext"']

    function_calls = parser.parse_string('#comment\na("# not comment") #comment \n ', parseAll=True)
    assert len(function_calls) == 1
    function_name, function_args = function_calls[0]
    assert function_name == "a"
    assert function_args == ["# not comment"]

    function_calls = parser.parse_string('fun(12, x="val", y={"aaa": 23})', parseAll=True)
    function_name, function_args = function_calls[0]
    assert function_name == "fun"
    assert function_args == [
        12,
        NamedArgument(name="x", value="val"),
        NamedArgument(name="y", value={"aaa": 23}),
    ]


def test_valid_action():
    action_set = HighLevelActionSet()

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": CHECKBOX_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    def get_checkbox_elem(obs):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        checkbox = soup.find("input", attrs={"type": "checkbox", "id": "vehicle1"})
        return checkbox

    obs, info = env.reset()
    checkbox = get_checkbox_elem(obs)

    # box not checked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # typo in action (unescaped double quotes)
    action = f"""\
click({repr(checkbox.get(BID_ATTR))}, "17" screen")  # typo here
"""
    with pytest.raises(ValueError):
        python_action = action_set.to_python_code(action)

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # error and box not checked
    assert "Received an empty action." in obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # click box 1 time
    action = f"""\
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 1

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box checked
    assert not obs["last_action_error"]
    assert checkbox.has_attr("checked")

    # click box 2 times
    action = f"""\
click({repr(checkbox.get(BID_ATTR))})
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 2

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box still checked
    assert not obs["last_action_error"]
    assert checkbox.has_attr("checked")

    # click box 3 times
    action = f"""\
click({repr(checkbox.get(BID_ATTR))})
click({repr(checkbox.get(BID_ATTR))})
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 3

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box unchecked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # click box 3 times, same line ops
    action = f"""\
click({repr(checkbox.get(BID_ATTR))}) click({repr(checkbox.get(BID_ATTR))}) click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 3

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box checked
    assert not obs["last_action_error"]
    assert checkbox.has_attr("checked")

    # click box 3 times, multi line ops, whitespace, tab, comma in-between args
    action = f"""\
   click(  {repr(checkbox.get(BID_ATTR))} ) click({repr(checkbox.get(BID_ATTR))})\t
                                                  noop() noop () noop( )
   # THIS IS A COMMENT
   noop() # this is a noop() call
click({repr(checkbox.get(BID_ATTR))},   )
#click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 3

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box unchecked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # click box 3 times, multi line ops, whitespace, tab, comma in-between args, markdown code block
    action = f"""\
Below is code
        ```python
   click(  {repr(checkbox.get(BID_ATTR))} ) click({repr(checkbox.get(BID_ATTR))})\t
                                                  noop() noop () noop( )
   # THIS IS A COMMENT
   noop() # this is a noop() call
click({repr(checkbox.get(BID_ATTR))},   )
#click({repr(checkbox.get(BID_ATTR))})
```
This is not code, just an explanation
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 3

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box checked
    assert not obs["last_action_error"]
    assert checkbox.has_attr("checked")

    # multiple markdown code blocks
    action = f"""\
Below is code
        ```python
                                                  noop() noop () noop( )
   # THIS IS A COMMENT
   noop() # this is a noop() call
click({repr(checkbox.get(BID_ATTR))},   )
#click({repr(checkbox.get(BID_ATTR))})
```
This is not code, just an explanation
Below is more code
        ```python
   click(  {repr(checkbox.get(BID_ATTR))} ) click({repr(checkbox.get(BID_ATTR))})\t
                                                  noop() noop () noop( )
   # THIS IS A COMMENT
   noop() # this is a noop() call
#click({repr(checkbox.get(BID_ATTR))})
```
This is not code, just an explanation
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 3

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box unchecked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # multiple function calls in the middle of text
    action = f"""\
Let's do a noop(), then noop () noop( ) then click({repr(checkbox.get(BID_ATTR))},   )
#click({repr(checkbox.get(BID_ATTR))})
Now let's do two more
   click(  {repr(checkbox.get(BID_ATTR))} ) click({repr(checkbox.get(BID_ATTR))})\t
                                                  noop() noop () noop( )
   # THIS IS A COMMENT
   noop() # this is a noop() call
#click({repr(checkbox.get(BID_ATTR))})
```
This is not code, just an explanation
This is garbage
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nclick(") == 3

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box checked
    assert not obs["last_action_error"]
    assert checkbox.has_attr("checked")

    env.close()


def test_invalid_action():
    action_set = HighLevelActionSet()

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": CHECKBOX_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )
    obs, info = env.reset()

    # click inexistant bid
    action = f"""\
click("INVALID_BID")
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert "ValueError" in obs["last_action_error"]

    # invalid bid value type
    action = f"""\
click(None)
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "ValueError: expected a string, got None"

    # invalid bid value type
    action = f"""\
click(42.7)
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "ValueError: expected a string, got 42.7"

    # invalid bid value type
    action = f"""\
click([])
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "ValueError: expected a string, got []"

    # invalid bid value type
    action = f"""\
click([42, "a", True, None])
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "ValueError: expected a string, got [42, 'a', True, None]"

    # invalid bid value type
    action = f"""\
click({{}})
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "ValueError: expected a string, got {}"

    # invalid bid value type
    action = f"""\
click({{"k": "aaa"}})
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "ValueError: expected a string, got {'k': 'aaa'}"

    # invalid action args (too many)
    action = f"""\
click("4", "aa", "bb")
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert obs["last_action_error"] == "Error: Locator.click: modifiers: expected array, got string"

    # invalid action args (not enough)
    action = f"""\
click()
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert (
        obs["last_action_error"]
        == "TypeError: click() missing 1 required positional argument: 'bid'"
    )

    # invalid action args (not enough)
    action = f"""\
click()
"""

    obs, reward, term, trunc, info = env.step(action)

    # error
    assert (
        obs["last_action_error"]
        == "TypeError: click() missing 1 required positional argument: 'bid'"
    )

    # invalid action name
    with pytest.raises(NameError):
        action_set.to_python_code(
            f"""\
not_a_valid_action()
"""
        )

    # forbidden fill action
    with pytest.raises(NameError):
        HighLevelActionSet(subsets=["coord"]).to_python_code(
            f"""\
fill("INVALID_BID", "some text")
"""
        )

    # forbidden import
    with pytest.raises(ValueError):
        action_set.to_python_code(
            f"""\
import numpy as np
"""
        )

    # invalid expression, results in empty action
    with pytest.raises(ValueError):
        action_set.to_python_code(
            f"""\
[
"""
        )

    # invalid expression, results in empty action
    with pytest.raises(ValueError):
        action_set.to_python_code(
            f"""\
click
"""
        )

    env.close()


def test_click_through_frames():
    action_set = HighLevelActionSet()

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": MULTI_IFRAME_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    obs, info = env.reset()

    soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
    checkbox = soup.find("input", attrs={"type": "checkbox", "id": "checkbox_2"})

    # box checked
    assert checkbox.has_attr("checked")

    # click box
    action = f"""\
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    obs, reward, term, trunc, info = env.step(action)

    # no error
    assert not obs["last_action_error"]

    soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
    checkbox = soup.find("input", attrs={"type": "checkbox", "id": "checkbox_2"})

    # box not checked
    assert not checkbox.has_attr("checked")

    env.close()


def test_fill_through_iframe():
    action_set = HighLevelActionSet()

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": MULTI_IFRAME_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    obs, info = env.reset()

    soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
    text_input = soup.find(
        "input", attrs={"type": "text", "placeholder": "Enter text here in iframe"}
    )

    # empty input
    assert text_input.get("value") == ""

    # fill with some text
    action = f"""\
fill({repr(text_input.get(BID_ATTR))}, "This is a test value.")
"""
    python_action = action_set.to_python_code(action)

    obs, reward, term, trunc, info = env.step(action)

    # no error
    assert not obs["last_action_error"]

    soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
    text_input = soup.find(
        "input", attrs={"type": "text", "placeholder": "Enter text here in iframe"}
    )

    # input filled to desired value
    assert text_input.get("value") == "This is a test value."

    env.close()


def test_click():
    action_set = HighLevelActionSet()

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": CHECKBOX_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    def get_checkbox_elem(obs):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        checkbox = soup.find("input", attrs={"type": "checkbox", "id": "vehicle1"})
        return checkbox

    obs, info = env.reset()
    checkbox = get_checkbox_elem(obs)

    # box not checked
    assert not checkbox.has_attr("checked")

    # click box
    action = f"""
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    obs, reward, terminated, truncated, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # no error
    assert not obs["last_action_error"]

    # box checked
    assert checkbox.has_attr("checked")

    # click box
    action = f"""\
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # no error
    assert not obs["last_action_error"]

    # box unchecked
    assert not checkbox.has_attr("checked")

    # click box twice
    action = f"""\
click({repr(checkbox.get(BID_ATTR))})
click({repr(checkbox.get(BID_ATTR))})
"""
    python_action = action_set.to_python_code(action)

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # no error
    assert not obs["last_action_error"]

    # box still unchecked
    assert not checkbox.has_attr("checked")

    env.close()


def test_hover():
    action_set = HighLevelActionSet(subsets=["bid", "coord"])

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": HOVER_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    def get_button_elem(obs):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        button = soup.find("input", attrs={"type": "button"})
        return button

    obs, info = env.reset()
    button = get_button_elem(obs)

    assert not obs["last_action_error"]
    assert button.get("value") == "Hover me"

    action = f"""
hover({repr(button.get(BID_ATTR))})
"""

    obs, reward, terminated, truncated, info = env.step(action)
    button = get_button_elem(obs)

    assert not obs["last_action_error"]
    assert button.get("value") == "Hello world!"

    action = f"""
mouse_move(0, 0)
"""

    obs, reward, terminated, truncated, info = env.step(action)
    button = get_button_elem(obs)

    assert not obs["last_action_error"]
    assert button.get("value") == "Hover me"

    env.close()


def test_fill_type_press():
    action_set = HighLevelActionSet(subsets=["bid", "coord"])
    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": TEXT_INPUT_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    def get_fname_lname_elems(obs):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        fname = soup.find("input", attrs={"id": "fname"})
        lname = soup.find("input", attrs={"id": "lname"})
        return fname, lname

    obs, info = env.reset()
    fname, lname = get_fname_lname_elems(obs)

    # type using bid
    action = f"""
fill({repr(fname.get(BID_ATTR))}, 'Christian')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "Christian"
    assert lname.get("value") == ""

    # type using bid
    action = f"""
fill({repr(lname.get(BID_ATTR))}, 'Clavier')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "Christian"
    assert lname.get("value") == "Clavier"

    # type using focus and keyboard_type
    action = f"""
focus({repr(fname.get(BID_ATTR))}) keyboard_type('Gérard')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "ChristianGérard"
    assert lname.get("value") == "Clavier"

    # type using click and keyboard_insert_text
    action = f"""
click({repr(lname.get(BID_ATTR))}) keyboard_insert_text('Jugnot')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "ChristianGérard"
    assert lname.get("value") == "ClavierJugnot"

    # type using clear and keyboard_insert_text
    action = f"""
clear({repr(lname.get(BID_ATTR))}) keyboard_insert_text('Jugnot')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "ChristianGérard"
    assert lname.get("value") == "Jugnot"

    # type using click, manual clear and keyboard_insert_text
    action = f"""
click({repr(fname.get(BID_ATTR))})
# clear the field
keyboard_press('End')
keyboard_down('Shift')
keyboard_press('Home')
keyboard_up('Shift')
keyboard_press('Backspace')
# insert text
keyboard_insert_text('Gérard')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "Gérard"
    assert lname.get("value") == "Jugnot"

    # fill empty text
    action = f"""
fill({repr(fname.get(BID_ATTR))}, '')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == ""
    assert lname.get("value") == "Jugnot"

    # type in currently focused element
    action = f"""
keyboard_type('Jean')
"""

    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "Jean"
    assert lname.get("value") == "Jugnot"

    # de-focus (click 0, 0), then type text
    action = f"""
mouse_click(0, 0)
"""
    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "Jean"
    assert lname.get("value") == "Jugnot"

    action = f"""
keyboard_type('Reno')
"""
    obs, reward, terminated, truncated, info = env.step(action)
    fname, lname = get_fname_lname_elems(obs)

    assert not obs["last_action_error"]
    assert fname.get("value") == "Jean"
    assert lname.get("value") == "Jugnot"

    env.close()


@pytest.mark.skip(reason="Not implemented yet")
def test_dblclick():
    pass


# copy/paste text using a sequence of keyboard_press actions
def test_key_press():
    action_set = HighLevelActionSet(subsets=["bid", "coord"])

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": TEXT_INPUT_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    obs, info = env.reset()

    def get_fname_lname_elems(obs):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        fname = soup.find("input", attrs={"id": "fname"})
        lname = soup.find("input", attrs={"id": "lname"})
        return fname, lname

    fname, lname = get_fname_lname_elems(obs)

    action = f"""
    fill({repr(fname.get(BID_ATTR))}, "Christian")
    keyboard_press({repr("Meta+a" if _IS_MAC_OS else "Control+a")})
    keyboard_press({repr("Meta+c" if _IS_MAC_OS else "Control+c")})
    click({repr(lname.get(BID_ATTR))})
    keyboard_press({repr("Meta+v" if _IS_MAC_OS else "Control+v")})
    """

    obs, reward, terminated, truncated, info = env.step(action)

    assert not obs["last_action_error"]

    fname, lname = get_fname_lname_elems(obs)

    assert lname.get("value") == "Christian"

    env.close()


def test_goto():
    url1 = URL_INPUT_URL
    url2 = TEXT_INPUT_URL

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": url1},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
    )

    obs, info = env.reset()

    assert obs["url"] == url1

    action = f"""
goto({repr(url2)})
"""

    obs, reward, terminated, truncated, info = env.step(action)

    assert not obs["last_action_error"]

    assert obs["url"] == url2

    action = """
go_back()
"""

    obs, reward, terminated, truncated, info = env.step(action)

    assert not obs["last_action_error"]

    assert obs["url"] == url1

    action = """
go_forward()
"""

    obs, reward, terminated, truncated, info = env.step(action)

    assert not obs["last_action_error"]

    assert obs["url"] == url2

    env.close()


def test_scroll():
    action_set = HighLevelActionSet(subsets=["coord"])

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": LONG_PAGE_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    def extract_coords_from_elem(elem):
        return ast.literal_eval(elem.get("center"))

    def get_top_bottom_elems(obs):
        soup = bs4.BeautifulSoup(
            flatten_dom_to_str(
                obs["dom_object"], obs["extra_element_properties"], with_center_coords=True
            ),
            "lxml",
        )
        top = soup.find("input", attrs={"type": "checkbox", "id": "top"})
        bottom = soup.find("input", attrs={"type": "checkbox", "id": "bottom"})
        return top, bottom

    obs, info = env.reset()
    top, bottom = get_top_bottom_elems(obs)
    top_x, top_y = extract_coords_from_elem(top)
    bottom_x, bottom_y = extract_coords_from_elem(bottom)

    # top not checked
    assert not top.has_attr("checked")
    # bottom not checked
    assert not bottom.has_attr("checked")

    # click top
    action = f"mouse_click({repr(top_x)}, {repr(top_y)})"

    obs, reward, terminated, truncated, info = env.step(action)

    top, bottom = get_top_bottom_elems(obs)
    top_x, top_y = extract_coords_from_elem(top)
    bottom_x, bottom_y = extract_coords_from_elem(bottom)

    # no error
    assert not obs["last_action_error"]
    # top checked
    assert top.has_attr("checked")
    # bottom not checked
    assert not bottom.has_attr("checked")

    top, bottom = get_top_bottom_elems(obs)
    top_x, top_y = extract_coords_from_elem(top)
    bottom_x, bottom_y = extract_coords_from_elem(bottom)

    # click bottom
    action = f"mouse_click({repr(bottom_x)}, {repr(bottom_y)})"

    obs, reward, terminated, truncated, info = env.step(action)

    top, bottom = get_top_bottom_elems(obs)
    top_x, top_y = extract_coords_from_elem(top)
    bottom_x, bottom_y = extract_coords_from_elem(bottom)

    # no error (click coordinates out of viewport is a silent fail in playwright)
    assert not obs["last_action_error"]
    # top checked
    assert top.has_attr("checked")
    # bottom not checked (click didn't go through)
    assert not bottom.has_attr("checked")

    # scroll up
    action = f"scroll(0, -500)"

    obs, reward, terminated, truncated, info = env.step(action)

    top, bottom = get_top_bottom_elems(obs)
    prev_top_x, prev_top_y = top_x, top_y
    top_x, top_y = extract_coords_from_elem(top)
    prev_bottom_x, prev_bottom_y = bottom_x, bottom_y
    bottom_x, bottom_y = extract_coords_from_elem(bottom)

    # no error
    assert not obs["last_action_error"]

    # no movement
    assert prev_top_x == top_x and prev_top_y == top_y
    assert prev_bottom_x == bottom_x and prev_bottom_y == bottom_y

    # scroll down
    action = f"scroll(0, 500)"

    obs, reward, terminated, truncated, info = env.step(action)

    top, bottom = get_top_bottom_elems(obs)
    prev_top_x, prev_top_y = top_x, top_y
    top_x, top_y = extract_coords_from_elem(top)
    prev_bottom_x, prev_bottom_y = bottom_x, bottom_y
    bottom_x, bottom_y = extract_coords_from_elem(bottom)

    # no error
    assert not obs["last_action_error"]

    # movement
    assert prev_top_x == top_x and prev_top_y > top_y
    assert prev_bottom_x == bottom_x and prev_bottom_y > bottom_y

    env.close()


def test_tab_actions():
    action_set = HighLevelActionSet(subsets=["tab", "nav"])

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": CHECKBOX_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )
    obs, info = env.reset()
    assert not obs["last_action_error"]
    assert len(obs["open_pages_urls"]) == 1
    assert len(obs["open_pages_titles"]) == 1
    assert obs["active_page_index"] == 0
    assert obs["open_pages_urls"][obs["active_page_index"][0]] == obs["url"]

    obs, reward, terminated, truncated, info = env.step("new_tab()")
    assert not obs["last_action_error"]
    assert len(obs["open_pages_urls"]) == 2
    assert len(obs["open_pages_titles"]) == 2
    assert obs["active_page_index"] == 1
    assert obs["open_pages_urls"][obs["active_page_index"][0]] == obs["url"]

    obs, reward, terminated, truncated, info = env.step(f"goto({repr(TEXTBOX_URL)})")
    assert not obs["last_action_error"]
    assert len(obs["open_pages_urls"]) == 2
    assert len(obs["open_pages_titles"]) == 2
    assert obs["active_page_index"] == 1
    assert obs["open_pages_urls"][obs["active_page_index"][0]] == obs["url"]

    obs, reward, terminated, truncated, info = env.step("tab_focus(0)")
    assert not obs["last_action_error"]
    assert len(obs["open_pages_urls"]) == 2
    assert len(obs["open_pages_titles"]) == 2
    assert obs["active_page_index"] == 0
    assert obs["open_pages_urls"][obs["active_page_index"][0]] == obs["url"]

    obs, reward, terminated, truncated, info = env.step("tab_close()")
    assert not obs["last_action_error"]
    assert len(obs["open_pages_urls"]) == 1
    assert len(obs["open_pages_titles"]) == 1
    assert obs["active_page_index"] == 0
    assert obs["open_pages_urls"][obs["active_page_index"][0]] == obs["url"]

    env.close()


def test_mouse_down_up():
    action_set = HighLevelActionSet(subsets=["bid", "coord"])

    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": CHECKBOX_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    def get_checkbox_elem(obs):
        soup = bs4.BeautifulSoup(
            flatten_dom_to_str(
                obs["dom_object"], obs["extra_element_properties"], with_center_coords=True
            ),
            "lxml",
        )
        checkbox = soup.find("input", attrs={"type": "checkbox", "id": "vehicle1"})
        return checkbox

    obs, info = env.reset()
    checkbox = get_checkbox_elem(obs)

    # box not checked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # click box 1 time
    x, y = ast.literal_eval(checkbox.get("center"))
    action = f"""\
mouse_click({repr(x)}, {repr(y)})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nmouse_") == 1

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box checked
    assert not obs["last_action_error"]
    assert checkbox.has_attr("checked")

    # click box 1 time
    x, y = ast.literal_eval(checkbox.get("center"))
    action = f"""\
mouse_move(0, 0)
mouse_move({repr(x)}, {repr(y)})
mouse_down({repr(x)}, {repr(y)})
mouse_up({repr(x)}, {repr(y)})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nmouse_") == 4

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box not checked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")

    # click box 2 times
    x, y = ast.literal_eval(checkbox.get("center"))
    action = f"""\
mouse_move(0, 0)
mouse_move({repr(x)}, {repr(y)})
mouse_down({repr(x)}, {repr(y)}, button="left")
mouse_up({repr(x)}, {repr(y)}, "left")
mouse_down({repr(x)}, {repr(y)})
mouse_up({repr(x)}, {repr(y)})
"""
    python_action = action_set.to_python_code(action)

    assert python_action.count("\nmouse_") == 6

    obs, reward, term, trunc, info = env.step(action)
    checkbox = get_checkbox_elem(obs)

    # box not checked
    assert not obs["last_action_error"]
    assert not checkbox.has_attr("checked")


# test that forced action can click an obstructed element
@pytest.mark.parametrize("retry_with_force", [True, False])
def test_forced_actions(retry_with_force):
    action_set = HighLevelActionSet(subsets=["bid"], retry_with_force=retry_with_force)
    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": OBSTRUCTED_CHECKBOX_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    obs, info = env.reset()

    def get_checkbox(obs):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        checkbox = soup.find("input", attrs={"id": "hobbies-checkbox-1"})
        return checkbox

    checkbox = get_checkbox(obs)

    action = f"""
    click({repr(checkbox.get(BID_ATTR))})
    """

    obs, reward, terminated, truncated, info = env.step(action)
    checkbox = get_checkbox(obs)
    if retry_with_force:
        assert not obs["last_action_error"]
        assert checkbox.get("checked", False) == False
    else:
        assert obs["last_action_error"]
        assert checkbox.has_attr("checked")

    env.close()


# TODO investigate why it takes ~1sec to mark each frame, although they are very small, and if we can do something about it
@pytest.mark.slow
def test_iframe_bid():
    action_set = HighLevelActionSet(subsets=["bid"])
    env = gym.make(
        "browsergym/openended",
        task_kwargs={"start_url": LOTS_OF_IFRAMES_URL},
        headless=__HEADLESS,
        slow_mo=__SLOW_MO,
        timeout=__TIMEOUT,
        action_mapping=action_set.to_python_code,
    )

    obs, info = env.reset()

    def get_checkbox(obs, i):
        soup = bs4.BeautifulSoup(flatten_dom_to_str(obs["dom_object"]), "lxml")
        checkbox = soup.find("input", attrs={"id": f"checkbox{i}"})
        return checkbox

    # try to click on checkboxes
    checkboxes = [
        (0, "a"),
        # (5, "f"),
        # (26, "aA"),
        (29, "aD"),
    ]
    for id, iframe_bid in checkboxes:

        # try to click on checkbox
        checkbox = get_checkbox(obs, id)
        bid = checkbox.get(BID_ATTR)

        # iframe bid should match
        assert re.match(f"^{iframe_bid}[0-9]+$", bid)

        action = f"""
        click({repr(bid)})
        """

        obs, reward, terminated, truncated, info = env.step(action)
        assert not obs["last_action_error"]

        # checkbox should get checked
        checkbox = get_checkbox(obs, id)
        assert checkbox.has_attr("checked")

    env.close()