File size: 3,083 Bytes
adf912b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """Direct diagnostic: can laya pick the right element on a real Wikipedia element table, under different prompt formats?"""
import json, os, random, sys, time
import httpx
sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast")
os.environ["BU_CDP_URL"] = "http://127.0.0.1:9222"
from jev_ultrafast.browser import Browser
from jev_ultrafast.model import action_space
from jev_ultrafast.questions import NEXT_ACTION, TARGET
S1 = "http://127.0.0.1:8791/v1/systemone"
page = None
def get_page():
global page
if page is None:
b = Browser("https://en.wikipedia.org/wiki/Main_Page"); page = b.observe(screenshot=False); b.close()
return page
GOALS = [("Click the 'Random article' link in the navigation.", "Random article"),
("Open the site's search box so a query can be typed.", "Search"),
("Log in to Wikipedia.", "Log in"),
("Open the Talk page for the main page.", "Talk"),
("Read the community portal.", "Community portal")]
def ask(state, questions):
r = httpx.post(S1, json={"model": "x", "state": state, "questions": questions}, timeout=120).json()
return r["answers"]
def run(fmt, K=1):
p = get_page(); elements, targets, controls = action_space(p["actions"])
cands = targets["CLICK"]
hits, ranks = 0, []
for goal, want in GOALS:
gold = [i for i, a in cands.items() if a["label"].split(" → ")[0].strip().lower() == want.lower()]
if not gold: print(" no gold for", want); continue
if fmt == "jev":
crit = {i: {"element": f"[{i}] {a['label']}", "current_value": a.get("current_value", a.get("value", "")), **{k: a[k] for k in ("role", "checked", "selected", "expanded") if k in a}} for i, a in cands.items()}
state = {"page": {k: p[k] for k in ("url", "title", "text")}, "elements": elements, "recent_actions": []}
ins = {"goal": goal, "operation": "CLICK", "rules": [NEXT_ACTION, TARGET]}
else:
crit = {i: a["label"].split(" → ")[0][:60] for i, a in cands.items()}
state = {"goal": goal, "page_title": p["title"], "url": p["url"]}
ins = f"Goal: {goal} Which element should be clicked next?"
probs = {i: 0.0 for i in crit}
for k in range(K):
keys = list(crit); random.Random(k).shuffle(keys) if K > 1 else None
a = ask(state, {"t": {"type": "choice", "instructions": ins, "criteria": {i: crit[i] for i in keys}}})["t"]
for i, v in a["probabilities"].items(): probs[i] += v / K
order = sorted(probs, key=probs.get, reverse=True)
rank = min(order.index(g) for g in gold) + 1
ranks.append(rank); hits += rank == 1
print(f" {want:18s} gold={gold} top={order[:3]} p_gold={max(probs[g] for g in gold):.3f} rank={rank}")
print(f" => top1 {hits}/{len(ranks)} mean rank {sum(ranks)/len(ranks):.1f} of {len(cands)}")
fmt = sys.argv[1]; K = int(sys.argv[2]) if len(sys.argv) > 2 else 1
print(f"== format={fmt} permutations={K} variant={httpx.get('http://127.0.0.1:8791/').json()['variant']}")
run(fmt, K)
|