adriandj3's picture
TernaryPilot v2: card and training kit (weights after training)
5c251fb verified
Raw History Blame Contribute Delete
11.4 kB
"""make_data_v2.py: TernaryPilot v2 training data, same format and prompts as v1 (make_data.py), more and different sources:
- GUI-360 (vyokky/GUI-360, MIT): Word/Excel/PowerPoint on Windows. Grounding (agent thought -> point) and action
prediction (task + history + screen -> thought + action), joined by screenshot. API actions (19%) are skipped.
- AgentNet (xlangai/AgentNet, MIT): every completed task with alignment >= 8, Windows AND macOS (v1: Windows, part 1 only).
- Multimodal-Mind2Web (OpenRAIL) and v1's click pool: through make_data.py (--v1-jsonl merges an existing v1 file).
Coordinates are integers on a 0-1000 grid over the image, as in v1.
python make_data_v2.py --gui360 DIR --agentnet-jsonl F --agentnet-images DIR [--v1-jsonl data/train.jsonl] \
--out data/train_v2.jsonl --max-gui360 40000 --max-agentnet 20000 [--exclude "bench/**/*.png"]
"""
import argparse
import glob
import json
import multiprocessing
import os
import random
import re
from PIL import Image
import make_data as md
_V = [] # dhash of the benchmark images, set before the filtering pool forks
def _hash(path):
return md.dhash(Image.open(path))
def _libera(path): # the image is not a near copy of any benchmark image
h = _hash(path)
return not any(bin(h ^ v).count("1") <= 12 for v in _V)
def filtra(cands, path, max_n, pool):
"""cands already shuffled: the first max_n whose image is free, checked in parallel a slice at a time"""
out = []
for i in range(0, len(cands), 2048):
parte = cands[i:i + 2048]
out += [c for c, ok in zip(parte, pool.map(_libera, [path(c) for c in parte], chunksize=16)) if ok]
if len(out) >= max_n:
break
return out[:max_n]
def norm(x, y, w, h):
return round(min(max(x / w, 0), 0.999) * 1000), round(min(max(y / h, 0), 0.999) * 1000)
def gui360_azione(js, w, h):
"""GUI-360 <tool_call>{function, args} -> our action text, or None (API actions and unknown ones are skipped)"""
f, a = js.get("function"), js.get("args") or {}
for k in ("coordinate", "start_coordinate", "end_coordinate"): # some steps carry null coordinates: skip them
c = a.get(k)
if c is not None and not (isinstance(c, (list, tuple)) and len(c) >= 2
and all(isinstance(v, (int, float)) for v in c[:2])):
return None
def p(c):
return "x=%d, y=%d" % norm(c[0], c[1], w, h)
if f == "click" and a.get("coordinate"):
b = a.get("button", "left")
nome = "right_click" if b == "right" else "double_click" if a.get("double") else "click"
return f"{nome}({p(a['coordinate'])})"
if f == "type":
testo = a.get("text", a.get("keys", ""))
if not isinstance(testo, str):
return None
keys = re.fullmatch(r"\{VK_(\w+)\}(\w)", testo)
pre = f"click({p(a['coordinate'])})\n" if a.get("coordinate") else ""
if keys:
return pre + 'hotkey("%s+%s")' % (keys.group(1).lower().replace("control", "ctrl"), keys.group(2).lower())
if re.fullmatch(r"\{(ENTER|TAB|ESC|DELETE|BACKSPACE)\}", testo):
return pre + 'press("%s")' % testo.strip("{}").lower()
return pre + "write(%s)" % json.dumps(testo, ensure_ascii=False)
if f == "drag" and a.get("start_coordinate") and a.get("end_coordinate"):
return f"move({p(a['start_coordinate'])})\ndrag({p(a['end_coordinate'])})"
if f == "wheel_mouse_input" and a.get("coordinate") is not None:
return f"move({p(a['coordinate'])})\nscroll({int(a.get('wheel_dist', 0))})"
return None
def gui360(cartella, rnd, pool, max_n):
"""blocks of examples from processed_data/{grounding_resize, action_prediction_train_resize}"""
gr = json.load(open(f"{cartella}/processed_data/grounding_resize/training_data.json", encoding="utf-8"))
ap = json.load(open(f"{cartella}/processed_data/action_prediction_train_resize/training_data.json", encoding="utf-8"))
pensiero, punto = {}, {}
for r in gr: # thought -> point, keyed by screenshot
img = r["images"][0].replace("\\", "/")
m = re.search(r"The instruction is:\n(.*?)\n\nOutput the coordinate", r["conversation"][0]["value"], re.S)
c = re.search(r"\[(\d+),\s*(\d+)\]", r["conversation"][1]["value"])
if m and c:
pensiero[img] = md.frase(m.group(1), 300)
punto[img] = (int(c.group(1)), int(c.group(2)))
traiett = {}
for r in ap: # group steps by trajectory (id = <...>_<traj>_<step>)
img = r["images"][0].replace("\\", "/")
tid, step = r["id"].rsplit("_", 1)
q = re.search(r"The instruction is:\n(.*?)\n\nThe history", r["conversation"][0]["value"], re.S)
t = re.search(r"<tool_call>\s*(\{.*\})\s*</tool_call>", r["conversation"][1]["value"], re.S)
if q and t:
try:
traiett.setdefault(tid, []).append((int(step), img, q.group(1).strip(), json.loads(t.group(1))))
except ValueError:
pass
cands = []
radice = f"{cartella}/processed_data/action_prediction_train_resize"
gradice = f"{cartella}/processed_data/grounding_resize"
for tid, passi in traiett.items():
storia = []
for step, img, compito, js in sorted(passi):
path = os.path.join(radice, img)
if not os.path.exists(path):
continue
w, h = Image.open(path).size # header only
act = gui360_azione(js, w, h)
th = pensiero.get(img)
if act:
if th:
gp = os.path.join(gradice, img)
gp = gp if img in punto and os.path.exists(gp) else None
cands.append((path, compito, list(storia), act, th, gp, punto.get(img)))
storia.append(md.frase(th or act.splitlines()[0], 160))
rnd.shuffle(cands)
print("gui360 candidates", len(cands), flush=True)
blocchi = []
for path, compito, storia, act, th, gp, pt in filtra(cands, lambda c: c[0], max_n, pool):
b = [{"cache": md.md5(path), "img": path, "user": md.p_azione("Windows computer", compito, storia),
"risposta": f"Thought: {th}\nAction: {act}", "tipo": "action_gui360"}]
if gp:
gw, gh = Image.open(gp).size
x, y = norm(*pt, gw, gh)
b.append({"cache": md.md5(gp), "img": gp, "user": md.P_CLIC + th, "risposta": '{"x": %d, "y": %d}' % (x, y),
"tipo": "click_gui360"})
blocchi.append(b)
return blocchi
def agentnet(jsonl, cartella, rnd, pool, max_n):
mac = re.compile(r"\bmac ?os\b|\bfinder\b|\bdock\b|\bmenu bar\b|\bcommand key|\bcmd\b|safari|spotlight", re.I)
passi = []
for riga in open(jsonl, encoding="utf-8"):
d = json.loads(riga)
if not d.get("task_completed") or (d.get("alignment_score") or 0) < 8:
continue
oss = " ".join(s["value"].get("observation", "") for s in d["traj"][:3])
dove = "Mac computer" if mac.search(oss) else "Windows computer"
compito, storia = d.get("natural_language_task") or d["instruction"], []
for s in d["traj"]:
v, img = s["value"], os.path.join(cartella, s["image"])
act = md.converti(v.get("code") or "")
if act and os.path.exists(img) and v.get("last_step_correct", True) is not False:
passi.append((img, dove, compito, list(storia), v, act))
storia.append(md.frase(v.get("action", ""), 160))
rnd.shuffle(passi)
blocchi = []
for img, dove, compito, storia, v, act in filtra(passi, lambda p: p[0], max_n, pool):
k = md.md5(img)
b = [{"cache": k, "img": img, "user": md.p_azione(dove, compito, storia),
"risposta": f"Thought: {md.frase(v.get('action', ''), 220)}\nAction: {act}", "tipo": "action_pc"}]
if rnd.random() < 0.25 and v.get("observation"):
b.append({"cache": k, "img": img, "user": md.P_LETTURA, "risposta": md.frase(v["observation"], 900), "tipo": "reading"})
mm = re.fullmatch(r"(?:double_|right_)?click\((x=\d+, y=\d+)\)", act)
t = re.sub(r"^(double[- ]|right[- ])?click (on |the )*", "", md.frase(v.get("action", ""), 120), flags=re.I).rstrip(".")
if mm and 3 <= len(t) <= 100:
x, y = re.findall(r"\d+", mm.group(1))
b.append({"cache": k, "img": img, "user": md.P_CLIC + t, "risposta": '{"x": %s, "y": %s}' % (x, y), "tipo": "click_pc"})
blocchi.append(b)
return blocchi
def v1_blocchi(path): # an existing v1 train.jsonl, regrouped by screenshot
per = {}
for riga in open(path, encoding="utf-8"):
e = json.loads(riga)
per.setdefault(e["cache"], []).append(e)
return list(per.values())
def scrivi(blocchi, out, rnd): # same grouping as v1: blocks of one screenshot stay inside one window of 4
rnd.shuffle(blocchi)
finestre, aperte = [], []
for b in blocchi:
b = b[:4]
for f in aperte:
if sum(map(len, f)) + len(b) <= 4:
f.append(b)
break
else:
aperte.append([b])
for f in [f for f in aperte if sum(map(len, f)) == 4]:
finestre.append(f)
aperte.remove(f)
finestre += aperte
rnd.shuffle(finestre)
os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
tipi = {}
with open(out, "w", encoding="utf-8") as fo:
for f in finestre:
for b in f:
for e in b:
fo.write(json.dumps(e, ensure_ascii=False) + "\n")
tipi[e["tipo"]] = tipi.get(e["tipo"], 0) + 1
print("examples", sum(tipi.values()), "screenshots", len(blocchi), tipi)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--gui360"), ap.add_argument("--agentnet-jsonl", nargs="*"), ap.add_argument("--agentnet-images")
ap.add_argument("--v1-jsonl"), ap.add_argument("--exclude"), ap.add_argument("--out", required=True)
ap.add_argument("--max-gui360", type=int, default=40000), ap.add_argument("--max-agentnet", type=int, default=20000)
ap.add_argument("--seed", type=int, default=0)
a = ap.parse_args()
rnd = random.Random(a.seed)
n = max(1, (os.cpu_count() or 2) - 1)
if a.exclude:
with multiprocessing.Pool(n) as p:
_V[:] = p.map(_hash, glob.glob(a.exclude, recursive=True), chunksize=8)
print("excluded benchmark images", len(_V), flush=True)
pool = multiprocessing.Pool(n) # forked after _V is set, so the workers see it
blocchi = []
if a.gui360:
g = gui360(a.gui360, rnd, pool, a.max_gui360)
print("gui360 blocks", len(g), flush=True)
blocchi += g
for f in a.agentnet_jsonl or []:
g = agentnet(f, a.agentnet_images, rnd, pool, a.max_agentnet)
print("agentnet blocks", f, len(g), flush=True)
blocchi += g
if a.v1_jsonl:
g = v1_blocchi(a.v1_jsonl)
print("v1 blocks", len(g), flush=True)
blocchi += g
pool.close()
scrivi(blocchi, a.out, rnd)
if __name__ == "__main__":
main()