Spaces:
Running on Zero
Running on Zero
File size: 3,003 Bytes
d0e7c02 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | """Wave-UI (agentsea/wave-ui): UI elements with instructions and boxes, used as clicks in the agent
layout and as element boxes. The ScreenSpot source and every image that also appears in ScreenSpot
are removed, so ScreenSpot stays a clean test set."""
import hashlib
from . import load_cache, save_cache
# sources whose "instruction" is element-type metadata; their element name reads like an instruction
NAME_SOURCES = ("webui", "roboflow", "mind2web_test_domain", "mind2web_test_task", "mind2web_test_website")
def screenspot_hashes():
from datasets import load_dataset
key = lambda im: hashlib.md5(im.convert("L").resize((64, 64)).tobytes()).hexdigest()
return key, {key(im) for im in load_dataset("bevaya/ScreenSpot", split="test")["image"]}
def clean(split, n, seed, key, banned):
from datasets import load_dataset
ds = load_dataset("agentsea/wave-ui", split=split).shuffle(seed=seed)
ds = ds.filter(lambda src: src != "screenspot", input_columns=["source"])
out = []
for r in ds:
if len(out) >= n:
break
x1, y1, x2, y2 = r["bbox"]
W, H = r["image"].size
if not r["instruction"] or x2 <= x1 or y2 <= y1 or x2 > W * 1.02 or y2 > H * 1.02:
continue
if key(r["image"]) in banned:
continue
if r["source"] in NAME_SOURCES:
r = dict(r, instruction=r["name"])
if not r["instruction"]:
continue
out.append(r)
return out
def clicks(n, skip=0, cache=True):
"""Wave-UI elements as click steps in the agent layout (keeps the pointing sharp)."""
name = f"waveui_clicks_{n}_{skip}"
if cache and (got := load_cache(name)) is not None:
return got
key, banned = screenspot_hashes()
out = []
for r in clean("train", skip + n, 0, key, banned)[skip:]:
W, H = r["image"].size
x1, y1, x2, y2 = r["bbox"]
out.append(dict(image=r["image"], user=f"Instruction: {r['instruction']}\n\nPrevious actions:\nNone",
action="click", source="wave-ui",
args=dict(x=(x1 + x2) / 2 / W, y=(y1 + y2) / 2 / H, box=(x2 - x1) / W, boy=(y2 - y1) / H)))
if cache:
save_cache(name, out)
return out
def boxes(n, skip=2000, cache=True):
"""Wave-UI element boxes (pixels). Rows [skip, skip + n) are disjoint from the clicks [0, n_click)."""
name = f"waveui_boxes_{n}_{skip}"
if cache and (got := load_cache(name)) is not None:
return got
key, banned = screenspot_hashes()
out = [dict(image=r["image"], instruction=r["instruction"], box=list(r["bbox"]), source=r["source"])
for r in clean("train", skip + n, 0, key, banned)[skip:]]
if cache:
save_cache(name, out)
return out
def val_boxes(n):
key, banned = screenspot_hashes()
return [dict(image=r["image"], instruction=r["instruction"], box=list(r["bbox"]), source=r["source"])
for r in clean("validation", n, 1, key, banned)]
|