import gradio as gr
import subprocess, os, threading, torch, json, re, time, base64, io
subprocess.run(["playwright", "install", "chromium", "--with-deps"], capture_output=True)
from playwright.sync_api import sync_playwright
from transformers import AutoModelForCausalLM, AutoTokenizer
from PIL import Image
MODEL_ID = "Nanthasit/sakthai-coder-browser"
TOOLS = [
{"type": "function", "function": {"name": "navigate", "description": "Go to a URL", "parameters": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]}}},
{"type": "function", "function": {"name": "click", "description": "Click an element by selector or text", "parameters": {"type": "object", "properties": {"target": {"type": "string", "description": "CSS selector, text label, or coordinate"}}, "required": ["target"]}}},
{"type": "function", "function": {"name": "type", "description": "Type text into a field", "parameters": {"type": "object", "properties": {"selector": {"type": "string"}, "text": {"type": "string"}, "clear_first": {"type": "boolean"}}, "required": ["selector", "text"]}}},
{"type": "function", "function": {"name": "select", "description": "Select from dropdown", "parameters": {"type": "object", "properties": {"selector": {"type": "string"}, "value": {"type": "string"}}, "required": ["selector", "value"]}}},
{"type": "function", "function": {"name": "extract", "description": "Extract text content", "parameters": {"type": "object", "properties": {"selector": {"type": "string"}}}}},
{"type": "function", "function": {"name": "scroll", "description": "Scroll the page", "parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["up", "down", "top", "bottom"]}}, "required": ["direction"]}}},
{"type": "function", "function": {"name": "wait", "description": "Wait for milliseconds", "parameters": {"type": "object", "properties": {"ms": {"type": "integer"}}, "required": ["ms"]}}},
{"type": "function", "function": {"name": "press_key", "description": "Press a keyboard key", "parameters": {"type": "object", "properties": {"key": {"type": "string"}}, "required": ["key"]}}},
{"type": "function", "function": {"name": "screenshot", "description": "Take screenshot", "parameters": {"type": "object", "properties": {}}}},
{"type": "function", "function": {"name": "go_back", "description": "Go back", "parameters": {"type": "object", "properties": {}}}},
{"type": "function", "function": {"name": "done", "description": "Mark task as complete", "parameters": {"type": "object", "properties": {"answer": {"type": "string"}}}}},
]
print("Loading model on CPU...", flush=True)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map=None, torch_dtype=torch.float32)
print("Model loaded", flush=True)
_tls = threading.local()
def get_page():
if not hasattr(_tls, "page") or _tls.page is None:
pw = sync_playwright().start()
browser = pw.chromium.launch(headless=True, args=["--no-sandbox"])
_tls.page = browser.new_page(viewport={"width": 1280, "height": 800})
_tls.pw = pw
_tls.browser = browser
return _tls.page
def build_msgs(url, task):
fmt = (
"You are a browser automation agent. Complete the user's web task by outputting actions.\n"
"Output ONLY actions, one per line, in this exact format:\n"
"navigate(url=https://example.com)\n"
"click(target=#submit-button)\n"
"type(selector=#search, text=hello world)\n"
"select(selector=#country, value=TH)\n"
"extract(selector=.price)\n"
"scroll(direction=down)\n"
"wait(ms=2000)\n"
"press_key(key=Enter)\n"
"go_back()\n"
"done(answer=the result)\n"
"Example:\n"
f"User: Go to {url} and read the page title\n"
f"You: navigate(url={url})\n"
f"Tool: Page loaded: {url}. Page title: Example Domain\n"
"You: done(answer=Example Domain)\n"
)
return [
{"role": "system", "content": "You are a browser automation agent. Output tool-call lines only."},
{"role": "user", "content": fmt + f"\nTask: Go to {url} and {task}"},
]
def generate(msgs, max_new_tokens=160):
text = tokenizer.apply_chat_template(
msgs, tokenize=False, add_generation_prompt=True, tools=None,
)
inputs = tokenizer(text, return_tensors="pt")
with torch.no_grad():
out = model.generate(**inputs, max_new_tokens=max_new_tokens, temperature=0.1, do_sample=True)
return tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=False)
def generate_action(msgs):
return parse_actions(generate(msgs)), generate(msgs)
TOOL_NAME_MAP = {
"navigate": "navigate", "goto": "navigate", "open": "navigate",
"click": "click", "click_element": "click",
"type": "type", "fill": "type", "input": "type",
"select": "select", "select_option": "select",
"extract": "extract", "get_text": "extract", "read": "extract",
"scroll": "scroll", "wait": "wait", "sleep": "wait",
"press_key": "press_key", "key_press": "press_key",
"screenshot": "screenshot",
"go_back": "go_back", "back": "go_back",
"done": "done", "finish": "done", "complete": "done",
}
def clean_val(val):
val = str(val).strip()
val = val.split("<|im_end|>")[0].strip()
val = val.strip(" '\"").strip()
while val.endswith(")") and val.count(")") > val.count("("):
val = val[:-1].strip()
return val
def parse_actions(text):
actions = []
for m in re.finditer(r"(.*?)", text, re.DOTALL):
try:
data = json.loads(m.group(1).strip())
fn = data.get("function", data)
actions.append((fn.get("name", ""), fn.get("arguments", {})))
except json.JSONDecodeError:
pass
if not actions:
for m in re.finditer(r"\{[^{}]*\"name\"[^{}]*\}", text, re.DOTALL):
try:
data = json.loads(m.group(0))
fn = data.get("function", data)
args = fn.get("arguments", {})
if isinstance(args, str):
args = json.loads(args)
actions.append((fn.get("name", ""), args))
except json.JSONDecodeError:
pass
if not actions:
for line in text.splitlines():
line = line.strip().strip("`")
if not line or line.startswith(("<", "```")):
continue
m = re.match(r"([A-Za-z_]+)[,:]?\s*(.*)", line)
if not m:
continue
name_raw = m.group(1).lower()
name = TOOL_NAME_MAP.get(name_raw, name_raw)
rest = m.group(2)
if name not in TOOL_NAME_MAP.values():
continue
args = {}
if rest:
for kv in re.findall(r"(\w+)\s*=\s*([^,\n]+)", rest):
args[kv[0].strip()] = clean_val(kv[1])
if not args:
args = {"url": rest} if name == "navigate" else {"target": rest} if name == "click" else {"answer": clean_val(rest)} if name == "done" else {"text": rest}
if name == "navigate" and "url" not in args and "http" in rest:
args = {"url": rest.split()[0]}
actions.append((name, args))
return actions
def fix_selector(sel):
sel = str(sel).strip()
if sel.startswith("//") or sel.startswith("("):
return "xpath=" + sel
return sel
def run_tool(name, args, page):
if name == "navigate":
page.goto(args.get("url"), wait_until="domcontentloaded", timeout=30000)
return f"Navigated to {args.get('url')}"
elif name == "click":
target = fix_selector(args.get("target") or args.get("selector") or args.get("text"))
page.click(target, timeout=10000)
return f"Clicked {target}"
elif name == "type":
sel, text = fix_selector(args.get("selector") or args.get("target")), args.get("text")
if args.get("clear_first", True):
page.fill(sel, text)
else:
page.type(sel, text)
return f"Typed into {sel}"
elif name == "select":
page.select_option(fix_selector(args.get("selector")), args.get("value"))
return f"Selected {args.get('value')}"
elif name == "extract":
content = page.inner_text(fix_selector(args.get("selector"))) if args.get("selector") else page.inner_text("body")
return content[:800]
elif name == "scroll":
d = args.get("direction") or args.get("target") or "down"
d = d.split("<|im_end|>")[0].strip()
if d == "top": page.evaluate("window.scrollTo(0, 0);")
elif d == "bottom": page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
else:
delta = "window.innerHeight" if d == "down" else "-window.innerHeight"
page.evaluate(f"window.scrollBy(0, {delta});")
return f"Scrolled {d}"
elif name == "wait":
time.sleep(args.get("ms", 1000) / 1000)
return "Waited"
elif name == "press_key":
page.keyboard.press(args.get("key"))
return f"Pressed {args.get('key')}"
elif name == "screenshot":
return "Screenshot taken"
elif name == "go_back":
page.go_back(wait_until="domcontentloaded", timeout=30000)
return "Went back"
elif name == "done":
return "DONE:" + str(args.get("answer", ""))
return f"Unknown tool {name}"
def screenshot_png(page):
data = page.screenshot(type="png")
return Image.open(io.BytesIO(data))
def page_context(page):
try:
title = page.title()
body = page.inner_text("body")[:600]
return f"Page title: {title}\nPage text:\n{body}"
except Exception:
return ""
def run_task(url, task, max_steps=6):
page = get_page()
last_nav = None
shots = []
log = []
try:
page.goto(url, wait_until="domcontentloaded", timeout=30000)
last_nav = url
shots.append(screenshot_png(page))
log.append(f"OPEN {url} (page loaded)")
except Exception as e:
return "\n".join([f"ERROR navigating: {e}"]), None
msgs = [
{"role": "system", "content": "You are a browser automation agent. Output tool-call lines only."},
{"role": "user", "content": f"Task: Go to {url} and {task}"},
{"role": "assistant", "content": f"navigate(url={url})"},
{"role": "tool", "content": f"Page loaded: {url}\n{page_context(page)}", "name": "navigate"},
]
for step in range(max_steps):
raw = generate(msgs)
log.append(f"--- step {step+1}: model said ---\n{raw[:300]}")
actions = parse_actions(raw)
if not actions:
answer = raw.strip().strip("<|im_end|>").strip()
if answer and "{url}" not in answer and "> {name}({json.dumps(args, ensure_ascii=False)}) -> {result[:150]}")
msgs.append({"role": "assistant", "content": raw[:300]})
if name == "done":
answer = result[5:] if result.startswith("DONE:") else ""
if not answer:
answer = last_result if 'last_result' in dir() else ""
log.append(f"FINAL ANSWER: {answer[:300]}")
shots.append(screenshot_png(page))
return "\n".join(log), shots
last_result = result
msgs.append({"role": "tool", "content": result + "\n" + page_context(page), "name": name})
try:
shots.append(screenshot_png(page))
except Exception:
pass
return "\n".join(log), shots
with gr.Blocks(title="SakThai Web Agent") as demo:
gr.Markdown("# SakThai Web Agent")
gr.Markdown("Real Playwright browser on a free CPU Space. The agent drives headless Chromium and shows live screenshots of every step.")
with gr.Row():
url = gr.Textbox(value="https://www.google.com", label="URL", scale=2)
task = gr.Textbox(value="search for sakthai coder browser", label="Task", scale=3)
btn = gr.Button("Run in Playwright", variant="primary")
log_out = gr.Textbox(label="Agent log", lines=12)
gallery = gr.Gallery(label="Browser screenshots (after each action)", columns=3, height=400)
btn.click(fn=run_task, inputs=[url, task], outputs=[log_out, gallery])
demo.launch(show_error=True)