File size: 11,498 Bytes
3f84c1c
 
 
ca44759
 
3f84c1c
 
ca44759
 
 
 
 
 
3f84c1c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ca44759
 
 
 
 
 
 
 
 
 
 
 
 
 
3f84c1c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ca44759
 
3f84c1c
 
ca44759
3f84c1c
 
 
 
 
 
 
ca44759
 
 
3f84c1c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ca44759
3f84c1c
ca44759
 
 
3f84c1c
ca44759
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
"""Minimal reference harness for KORMo-Web-9B (see ../HARNESS.md).

    pip install openai playwright && playwright install chromium
    vllm serve KORMo-VL/KORMo-Web-9B --served-model-name KORMo-Web-9B \
        --max-model-len 131072 --limit-mm-per-prompt '{"image":1,"video":0}'
    python agent_playwright.py --task "Search for 'KAIST' on Google and open the official homepage."

  vLLM      : --tool-choice none (default). The tool schema is still rendered into the prompt
              and the call comes back as text in message.content.
  vllm-mlx  : --tool-choice auto. vllm-mlx drops the schema on "none" and returns the call in
              message.tool_calls; it is turned back into the <tool_call> XML below.
  --headful : show the browser window (default: headless).

It reproduces the message layout the model was trained on:
  system  = system_prompt.md          tools = tools.json (22 tools, rendered by chat_template.jinja)
  user    = <user_request>…</user_request> + <observation> (+ screenshot on the first step)
  then per step: assistant (raw <think>…</think> + <tool_call>) β†’ tool (feedback + <observation>)
  Full action/observation history is kept; only the *latest* observation carries a screenshot.
"""
from __future__ import annotations

import argparse
import asyncio
import base64
import json
import re
from pathlib import Path

from openai import OpenAI
from playwright.async_api import async_playwright

ROOT = Path(__file__).resolve().parent.parent
SYSTEM = (ROOT / "system_prompt.md").read_text(encoding="utf-8").strip()
TOOLS = json.loads((ROOT / "tools.json").read_text(encoding="utf-8"))
SPEC = {t["function"]["name"]: t["function"]["parameters"] for t in TOOLS}
TERMINAL = {"finished", "fail", "EXIT"}
SAMPLING = dict(temperature=0.6, top_p=0.95, max_tokens=16384,
                extra_body={"top_k": 20, "repetition_penalty": 1.0,
                            "chat_template_kwargs": {"enable_thinking": True}})


# ── parsing: exactly one <tool_call> in Qwen XML form ───────────────────────────
def parse_call(text: str) -> tuple[str, dict]:
    body = re.sub(r"^.*</think>", "", text, flags=re.S)
    calls = re.findall(r"<tool_call>\s*(.*?)\s*</tool_call>", body, re.S)
    if len(calls) != 1:
        raise ValueError("exactly one tool call required")
    m = re.fullmatch(r"<function=([^>\s]+)>\s*(.*?)\s*</function>", calls[0], re.S)
    if not m or m[1] not in SPEC:
        raise ValueError("malformed or unknown tool call")
    name, props, args = m[1], SPEC[m[1]]["properties"], {}
    for key, value in re.findall(r"<parameter=([^>\s]+)>\n?(.*?)\n?</parameter>", m[2], re.S):
        args[key] = value if props.get(key, {}).get("type") == "string" else json.loads(value)
    missing = [k for k in SPEC[name].get("required", []) if k not in args]
    if missing:
        raise ValueError(f"missing args {missing}")
    return name, args


# ── observation text (same layout as training) ─────────────────────────────────
async def observation(ctx, page, with_shot: bool) -> list[dict]:
    w, h = page.viewport_size["width"], page.viewport_size["height"]
    pages = ctx.pages
    lines = ["<observation>", f"screen size: {w} x {h}", f"Current tab: {pages.index(page)}", "Available tabs:"]
    for i, p in enumerate(pages):
        lines.append(f"- Tab {i}{' (active)' if p is page else ''}: {p.url[:300]}")
    lines += ["", f"url: {page.url[:300]}"]
    if not with_shot:
        return [{"type": "text", "text": "\n".join(lines) + "\n</observation>"}]
    png = base64.b64encode(await page.screenshot()).decode()
    return [{"type": "text", "text": "\n".join(lines) + "\nscreenshot:\n"},
            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{png}"}},
            {"type": "text", "text": "\n</observation>"}]


def restore_tool_calls(msg) -> str:
    """vllm-mlx returns the call in message.tool_calls; put it back as the <tool_call> XML the model wrote."""
    text = msg.content or ""
    if "<think>" not in text and "</think>" not in text and getattr(msg, "reasoning_content", None):
        text = f"<think>\n{msg.reasoning_content}\n</think>\n\n{text}"
    if getattr(msg, "tool_calls", None) and "<tool_call>" not in text:
        for tc in msg.tool_calls:
            args = json.loads(tc.function.arguments or "{}")
            params = "".join(f"<parameter={k}>\n{v if isinstance(v, str) else json.dumps(v, ensure_ascii=False)}\n</parameter>\n"
                             for k, v in args.items())
            text += f"\n\n<tool_call>\n<function={tc.function.name}>\n{params}</function>\n</tool_call>"
    return text


def drop_images(content: list[dict]) -> list[dict]:
    """Past turns keep the observation text but lose the screenshot."""
    out = [c for c in content if c["type"] == "text"]
    text = "".join(c["text"] for c in out).replace("\nscreenshot:\n\n</observation>", "\n</observation>")
    return [{"type": "text", "text": text}]


# ── execution: coordinates are 0-1000 normalized, scroll is in pixels ───────────
async def execute(ctx, page, name: str, a: dict):
    w, h = page.viewport_size["width"], page.viewport_size["height"]
    px = lambda x, y: (x / 1000 * w, y / 1000 * h)
    before, url0 = len(ctx.pages), page.url
    try:
        if name in ("click", "double_click", "right_click"):
            x, y = px(a["x"], a["y"])
            await page.mouse.click(x, y, button="right" if name == "right_click" else "left",
                                   click_count=2 if name == "double_click" else 1)
            fb = f"Succeed: `{name}` at ({a['x']}, {a['y']}) executed."
        elif name in ("hover", "focus"):
            x, y = px(a["x"], a["y"])
            await page.mouse.move(x, y)
            if name == "focus":
                await page.evaluate("([x,y]) => document.elementFromPoint(x,y)?.focus()", [x, y])
            fb = f"Succeed: `{name}` at ({a['x']}, {a['y']}) executed."
        elif name == "drag":
            await page.mouse.move(*px(a["x"], a["y"])); await page.mouse.down()
            await page.mouse.move(*px(a["x2"], a["y2"]), steps=10); await page.mouse.up()
            fb = "Succeed: `drag` executed."
        elif name == "type":
            await page.keyboard.type(a["text"], delay=30)
            fb = f'Succeed: `type` typed "{a["text"][:200]}".'
        elif name == "clear":
            await page.keyboard.press("Control+A"); await page.keyboard.press("Backspace")
            fb = "Succeed: `clear` executed."
        elif name == "hotkey":
            await page.keyboard.press("+".join(a["keys"]))
            fb = f"Succeed: `hotkey` {a['keys']} executed."
        elif name == "scroll":
            dx, dy = a.get("scroll_dx", 0), a["scroll_dy"]
            await page.mouse.wheel(dx, dy)
            fb = f"Succeed: `scroll` by (dx={dx}, dy={dy}) executed."
        elif name == "wait":
            await asyncio.sleep(float(a["seconds"]))
            fb = f"Succeed: waited {float(a['seconds'])} seconds."
        elif name == "goto":
            await page.goto(a["url"], timeout=60000)
            return page, f"Succeed: `goto` navigated to {page.url}."
        elif name in ("go_back", "go_forward"):
            await getattr(page, name)()
            fb = f"Succeed: `{name}` executed."
        elif name == "new_tab":
            page = await ctx.new_page()
            return page, f"Succeed: `new_tab` opened blank tab (tab {ctx.pages.index(page)})."
        elif name == "switch_tab":
            page = ctx.pages[int(a["index"])]; await page.bring_to_front()
            return page, f"Succeed: `switch_tab` switched to tab {a['index']} ({page.url})."
        elif name == "close_tab":
            await page.close(); page = ctx.pages[-1]
            return page, "Succeed: `close_tab` executed."
        elif name == "upload":
            return page, "Failed: `upload` execution failed: not supported by this harness"
        elif name == "unknown":
            return page, "No action executed."
        else:
            raise ValueError(name)
    except Exception as e:  # noqa: BLE001
        return page, f"Failed: `{name}` execution failed: {e}"
    await page.wait_for_timeout(1000)
    if len(ctx.pages) > before:                       # follow a newly opened tab
        page = ctx.pages[-1]; await page.bring_to_front()
        return page, fb + f" New tab opened (tab {len(ctx.pages) - 1})."
    if page.url != url0:
        return page, fb + f" Page navigated to {page.url}."
    return page, fb + (" Note: no visible navigation or new tab detected." if name.endswith("click") else "")


async def run(task: str, start_url: str, endpoint: str, model: str, max_steps: int,
              tool_choice: str = "none", headful: bool = False):
    client = OpenAI(base_url=endpoint, api_key="EMPTY")
    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=not headful)
        ctx = await browser.new_context(viewport={"width": 1280, "height": 720})
        page = await ctx.new_page()
        await page.goto(start_url)
        first = await observation(ctx, page, with_shot=True)
        first[0]["text"] = f"<user_request>\n{task}\n</user_request>\n" + first[0]["text"]
        msgs = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": first}]
        for step in range(max_steps):
            out = client.chat.completions.create(model=model, messages=msgs, tools=TOOLS,
                                                 tool_choice=tool_choice, **SAMPLING)
            text = restore_tool_calls(out.choices[0].message)
            msgs.append({"role": "assistant", "content": text})   # keep the full <think> + <tool_call>
            try:
                name, args = parse_call(text)
            except ValueError as e:
                fb = f"Failed: invalid tool call ({e})."
            else:
                print(f"[{step}] {name} {json.dumps(args, ensure_ascii=False)}")
                if name in TERMINAL:
                    print("DONE:", args.get("answer", name)); break
                page, fb = await execute(ctx, page, name, args)
            for m in msgs:                                       # only the newest observation keeps its screenshot
                if m["role"] in ("user", "tool") and isinstance(m["content"], list):
                    m["content"] = drop_images(m["content"])
            obs = await observation(ctx, page, with_shot=True)
            obs[0]["text"] = fb + "\n" + obs[0]["text"]
            msgs.append({"role": "tool", "content": obs})
        await browser.close()


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--task", required=True)
    ap.add_argument("--start-url", default="https://www.google.com/")
    ap.add_argument("--endpoint", default="http://localhost:8000/v1")
    ap.add_argument("--model", default="KORMo-Web-9B", help="the server's --served-model-name")
    ap.add_argument("--max-steps", type=int, default=50)
    ap.add_argument("--tool-choice", default="none", choices=["none", "auto"],
                    help="none for vLLM, auto for vllm-mlx (Apple silicon)")
    ap.add_argument("--headful", action="store_true", help="show the browser window")
    a = ap.parse_args()
    asyncio.run(run(a.task, a.start_url, a.endpoint, a.model, a.max_steps, a.tool_choice, a.headful))