m2w-gemma4-e2b-pointer / inference_example.py

Add LoRA adapter, pointer head, model card, code and results

a882958 verified 4 days ago

1.79 kB

"""Pick the next element on a live page with the released model.
Needs: a GPU, torch, transformers (Gemma 4 support), peft, playwright (`playwright install chromium`), lxml, scikit-learn, pillow.
The pointer and the confidence come from the model; typing the value and clicking is done by Playwright below.
Do not use the confidence to trigger irreversible actions: it is not calibrated, see the model card.
"""
import sys
sys.path.insert(0, "code")
from playwright.sync_api import sync_playwright
from m2w_form_demo import Gemma, aday_listesi, softmax

REPO = "senpaicq/m2w-gemma4-e2b-pointer"
URL = "https://example.com"
TASK = "Open the more information page"
HISTORY = []          # previous actions in the style "[link]  Flights -> CLICK" or "[textbox]  Where from? -> TYPE: LAX"

model = Gemma(REPO, shots_dir="tmp_shots")
with sync_playwright() as pw:
    browser = pw.chromium.launch(headless=True, args=["--no-sandbox"])
    page = browser.new_page(viewport={"width": 1280, "height": 720})
    page.goto(URL)
    cands, raw = aday_listesi(page, TASK, HISTORY, arm="etiketli")      # "etiketli": bound <label> text is added as the name
    png = page.screenshot(full_page=True)
    ex = {"task": TASK, "history": HISTORY, "cands": cands, "pos_idx": [], "op": "CLICK"}
    logits, op_logits = model(ex, png, None)                           # last logit = "none of these"
    p = softmax(logits, T=1.0)
    j = int(p.argmax())
    if j == len(cands):
        print("model abstains (none of the candidates)", float(p[j]))
    else:
        print("chosen:", cands[j]["text"], "| confidence:", round(float(p[j]), 3), "| operation logits [CLICK, TYPE, SELECT]:", op_logits.round(2))
        # el = page.query_selector(f'[data-m2w="{cands[j]["bid"]}"]'); el.click()
    browser.close()

	"""Pick the next element on a live page with the released model.

	Needs: a GPU, torch, transformers (Gemma 4 support), peft, playwright (`playwright install chromium`), lxml, scikit-learn, pillow.
	The pointer and the confidence come from the model; typing the value and clicking is done by Playwright below.
	Do not use the confidence to trigger irreversible actions: it is not calibrated, see the model card.
	"""
	import sys
	sys.path.insert(0, "code")
	from playwright.sync_api import sync_playwright
	from m2w_form_demo import Gemma, aday_listesi, softmax

	REPO = "senpaicq/m2w-gemma4-e2b-pointer"
	URL = "https://example.com"
	TASK = "Open the more information page"
	HISTORY = [] # previous actions in the style "[link] Flights -> CLICK" or "[textbox] Where from? -> TYPE: LAX"

	model = Gemma(REPO, shots_dir="tmp_shots")
	with sync_playwright() as pw:
	browser = pw.chromium.launch(headless=True, args=["--no-sandbox"])
	page = browser.new_page(viewport={"width": 1280, "height": 720})
	page.goto(URL)
	cands, raw = aday_listesi(page, TASK, HISTORY, arm="etiketli") # "etiketli": bound <label> text is added as the name
	png = page.screenshot(full_page=True)
	ex = {"task": TASK, "history": HISTORY, "cands": cands, "pos_idx": [], "op": "CLICK"}
	logits, op_logits = model(ex, png, None) # last logit = "none of these"
	p = softmax(logits, T=1.0)
	j = int(p.argmax())
	if j == len(cands):
	print("model abstains (none of the candidates)", float(p[j]))
	else:
	print("chosen:", cands[j]["text"], "\| confidence:", round(float(p[j]), 3), "\| operation logits [CLICK, TYPE, SELECT]:", op_logits.round(2))
	# el = page.query_selector(f'[data-m2w="{cands[j]["bid"]}"]'); el.click()
	browser.close()