File size: 1,788 Bytes
a882958 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Pick the next element on a live page with the released model.
Needs: a GPU, torch, transformers (Gemma 4 support), peft, playwright (`playwright install chromium`), lxml, scikit-learn, pillow.
The pointer and the confidence come from the model; typing the value and clicking is done by Playwright below.
Do not use the confidence to trigger irreversible actions: it is not calibrated, see the model card.
"""
import sys
sys.path.insert(0, "code")
from playwright.sync_api import sync_playwright
from m2w_form_demo import Gemma, aday_listesi, softmax
REPO = "senpaicq/m2w-gemma4-e2b-pointer"
URL = "https://example.com"
TASK = "Open the more information page"
HISTORY = [] # previous actions in the style "[link] Flights -> CLICK" or "[textbox] Where from? -> TYPE: LAX"
model = Gemma(REPO, shots_dir="tmp_shots")
with sync_playwright() as pw:
browser = pw.chromium.launch(headless=True, args=["--no-sandbox"])
page = browser.new_page(viewport={"width": 1280, "height": 720})
page.goto(URL)
cands, raw = aday_listesi(page, TASK, HISTORY, arm="etiketli") # "etiketli": bound <label> text is added as the name
png = page.screenshot(full_page=True)
ex = {"task": TASK, "history": HISTORY, "cands": cands, "pos_idx": [], "op": "CLICK"}
logits, op_logits = model(ex, png, None) # last logit = "none of these"
p = softmax(logits, T=1.0)
j = int(p.argmax())
if j == len(cands):
print("model abstains (none of the candidates)", float(p[j]))
else:
print("chosen:", cands[j]["text"], "| confidence:", round(float(p[j]), 3), "| operation logits [CLICK, TYPE, SELECT]:", op_logits.round(2))
# el = page.query_selector(f'[data-m2w="{cands[j]["bid"]}"]'); el.click()
browser.close()
|