"""make_data_v2.py: TernaryPilot v2 training data, same format and prompts as v1 (make_data.py), more and different sources: - GUI-360 (vyokky/GUI-360, MIT): Word/Excel/PowerPoint on Windows. Grounding (agent thought -> point) and action prediction (task + history + screen -> thought + action), joined by screenshot. API actions (19%) are skipped. - AgentNet (xlangai/AgentNet, MIT): every completed task with alignment >= 8, Windows AND macOS (v1: Windows, part 1 only). - Multimodal-Mind2Web (OpenRAIL) and v1's click pool: through make_data.py (--v1-jsonl merges an existing v1 file). Coordinates are integers on a 0-1000 grid over the image, as in v1. python make_data_v2.py --gui360 DIR --agentnet-jsonl F --agentnet-images DIR [--v1-jsonl data/train.jsonl] \ --out data/train_v2.jsonl --max-gui360 40000 --max-agentnet 20000 [--exclude "bench/**/*.png"] """ import argparse import glob import json import multiprocessing import os import random import re from PIL import Image import make_data as md _V = [] # dhash of the benchmark images, set before the filtering pool forks def _hash(path): return md.dhash(Image.open(path)) def _libera(path): # the image is not a near copy of any benchmark image h = _hash(path) return not any(bin(h ^ v).count("1") <= 12 for v in _V) def filtra(cands, path, max_n, pool): """cands already shuffled: the first max_n whose image is free, checked in parallel a slice at a time""" out = [] for i in range(0, len(cands), 2048): parte = cands[i:i + 2048] out += [c for c, ok in zip(parte, pool.map(_libera, [path(c) for c in parte], chunksize=16)) if ok] if len(out) >= max_n: break return out[:max_n] def norm(x, y, w, h): return round(min(max(x / w, 0), 0.999) * 1000), round(min(max(y / h, 0), 0.999) * 1000) def gui360_azione(js, w, h): """GUI-360 {function, args} -> our action text, or None (API actions and unknown ones are skipped)""" f, a = js.get("function"), js.get("args") or {} for k in ("coordinate", "start_coordinate", "end_coordinate"): # some steps carry null coordinates: skip them c = a.get(k) if c is not None and not (isinstance(c, (list, tuple)) and len(c) >= 2 and all(isinstance(v, (int, float)) for v in c[:2])): return None def p(c): return "x=%d, y=%d" % norm(c[0], c[1], w, h) if f == "click" and a.get("coordinate"): b = a.get("button", "left") nome = "right_click" if b == "right" else "double_click" if a.get("double") else "click" return f"{nome}({p(a['coordinate'])})" if f == "type": testo = a.get("text", a.get("keys", "")) if not isinstance(testo, str): return None keys = re.fullmatch(r"\{VK_(\w+)\}(\w)", testo) pre = f"click({p(a['coordinate'])})\n" if a.get("coordinate") else "" if keys: return pre + 'hotkey("%s+%s")' % (keys.group(1).lower().replace("control", "ctrl"), keys.group(2).lower()) if re.fullmatch(r"\{(ENTER|TAB|ESC|DELETE|BACKSPACE)\}", testo): return pre + 'press("%s")' % testo.strip("{}").lower() return pre + "write(%s)" % json.dumps(testo, ensure_ascii=False) if f == "drag" and a.get("start_coordinate") and a.get("end_coordinate"): return f"move({p(a['start_coordinate'])})\ndrag({p(a['end_coordinate'])})" if f == "wheel_mouse_input" and a.get("coordinate") is not None: return f"move({p(a['coordinate'])})\nscroll({int(a.get('wheel_dist', 0))})" return None def gui360(cartella, rnd, pool, max_n): """blocks of examples from processed_data/{grounding_resize, action_prediction_train_resize}""" gr = json.load(open(f"{cartella}/processed_data/grounding_resize/training_data.json", encoding="utf-8")) ap = json.load(open(f"{cartella}/processed_data/action_prediction_train_resize/training_data.json", encoding="utf-8")) pensiero, punto = {}, {} for r in gr: # thought -> point, keyed by screenshot img = r["images"][0].replace("\\", "/") m = re.search(r"The instruction is:\n(.*?)\n\nOutput the coordinate", r["conversation"][0]["value"], re.S) c = re.search(r"\[(\d+),\s*(\d+)\]", r["conversation"][1]["value"]) if m and c: pensiero[img] = md.frase(m.group(1), 300) punto[img] = (int(c.group(1)), int(c.group(2))) traiett = {} for r in ap: # group steps by trajectory (id = <...>__) img = r["images"][0].replace("\\", "/") tid, step = r["id"].rsplit("_", 1) q = re.search(r"The instruction is:\n(.*?)\n\nThe history", r["conversation"][0]["value"], re.S) t = re.search(r"\s*(\{.*\})\s*", r["conversation"][1]["value"], re.S) if q and t: try: traiett.setdefault(tid, []).append((int(step), img, q.group(1).strip(), json.loads(t.group(1)))) except ValueError: pass cands = [] radice = f"{cartella}/processed_data/action_prediction_train_resize" gradice = f"{cartella}/processed_data/grounding_resize" for tid, passi in traiett.items(): storia = [] for step, img, compito, js in sorted(passi): path = os.path.join(radice, img) if not os.path.exists(path): continue w, h = Image.open(path).size # header only act = gui360_azione(js, w, h) th = pensiero.get(img) if act: if th: gp = os.path.join(gradice, img) gp = gp if img in punto and os.path.exists(gp) else None cands.append((path, compito, list(storia), act, th, gp, punto.get(img))) storia.append(md.frase(th or act.splitlines()[0], 160)) rnd.shuffle(cands) print("gui360 candidates", len(cands), flush=True) blocchi = [] for path, compito, storia, act, th, gp, pt in filtra(cands, lambda c: c[0], max_n, pool): b = [{"cache": md.md5(path), "img": path, "user": md.p_azione("Windows computer", compito, storia), "risposta": f"Thought: {th}\nAction: {act}", "tipo": "action_gui360"}] if gp: gw, gh = Image.open(gp).size x, y = norm(*pt, gw, gh) b.append({"cache": md.md5(gp), "img": gp, "user": md.P_CLIC + th, "risposta": '{"x": %d, "y": %d}' % (x, y), "tipo": "click_gui360"}) blocchi.append(b) return blocchi def agentnet(jsonl, cartella, rnd, pool, max_n): mac = re.compile(r"\bmac ?os\b|\bfinder\b|\bdock\b|\bmenu bar\b|\bcommand key|\bcmd\b|safari|spotlight", re.I) passi = [] for riga in open(jsonl, encoding="utf-8"): d = json.loads(riga) if not d.get("task_completed") or (d.get("alignment_score") or 0) < 8: continue oss = " ".join(s["value"].get("observation", "") for s in d["traj"][:3]) dove = "Mac computer" if mac.search(oss) else "Windows computer" compito, storia = d.get("natural_language_task") or d["instruction"], [] for s in d["traj"]: v, img = s["value"], os.path.join(cartella, s["image"]) act = md.converti(v.get("code") or "") if act and os.path.exists(img) and v.get("last_step_correct", True) is not False: passi.append((img, dove, compito, list(storia), v, act)) storia.append(md.frase(v.get("action", ""), 160)) rnd.shuffle(passi) blocchi = [] for img, dove, compito, storia, v, act in filtra(passi, lambda p: p[0], max_n, pool): k = md.md5(img) b = [{"cache": k, "img": img, "user": md.p_azione(dove, compito, storia), "risposta": f"Thought: {md.frase(v.get('action', ''), 220)}\nAction: {act}", "tipo": "action_pc"}] if rnd.random() < 0.25 and v.get("observation"): b.append({"cache": k, "img": img, "user": md.P_LETTURA, "risposta": md.frase(v["observation"], 900), "tipo": "reading"}) mm = re.fullmatch(r"(?:double_|right_)?click\((x=\d+, y=\d+)\)", act) t = re.sub(r"^(double[- ]|right[- ])?click (on |the )*", "", md.frase(v.get("action", ""), 120), flags=re.I).rstrip(".") if mm and 3 <= len(t) <= 100: x, y = re.findall(r"\d+", mm.group(1)) b.append({"cache": k, "img": img, "user": md.P_CLIC + t, "risposta": '{"x": %s, "y": %s}' % (x, y), "tipo": "click_pc"}) blocchi.append(b) return blocchi def v1_blocchi(path): # an existing v1 train.jsonl, regrouped by screenshot per = {} for riga in open(path, encoding="utf-8"): e = json.loads(riga) per.setdefault(e["cache"], []).append(e) return list(per.values()) def scrivi(blocchi, out, rnd): # same grouping as v1: blocks of one screenshot stay inside one window of 4 rnd.shuffle(blocchi) finestre, aperte = [], [] for b in blocchi: b = b[:4] for f in aperte: if sum(map(len, f)) + len(b) <= 4: f.append(b) break else: aperte.append([b]) for f in [f for f in aperte if sum(map(len, f)) == 4]: finestre.append(f) aperte.remove(f) finestre += aperte rnd.shuffle(finestre) os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True) tipi = {} with open(out, "w", encoding="utf-8") as fo: for f in finestre: for b in f: for e in b: fo.write(json.dumps(e, ensure_ascii=False) + "\n") tipi[e["tipo"]] = tipi.get(e["tipo"], 0) + 1 print("examples", sum(tipi.values()), "screenshots", len(blocchi), tipi) def main(): ap = argparse.ArgumentParser() ap.add_argument("--gui360"), ap.add_argument("--agentnet-jsonl", nargs="*"), ap.add_argument("--agentnet-images") ap.add_argument("--v1-jsonl"), ap.add_argument("--exclude"), ap.add_argument("--out", required=True) ap.add_argument("--max-gui360", type=int, default=40000), ap.add_argument("--max-agentnet", type=int, default=20000) ap.add_argument("--seed", type=int, default=0) a = ap.parse_args() rnd = random.Random(a.seed) n = max(1, (os.cpu_count() or 2) - 1) if a.exclude: with multiprocessing.Pool(n) as p: _V[:] = p.map(_hash, glob.glob(a.exclude, recursive=True), chunksize=8) print("excluded benchmark images", len(_V), flush=True) pool = multiprocessing.Pool(n) # forked after _V is set, so the workers see it blocchi = [] if a.gui360: g = gui360(a.gui360, rnd, pool, a.max_gui360) print("gui360 blocks", len(g), flush=True) blocchi += g for f in a.agentnet_jsonl or []: g = agentnet(f, a.agentnet_images, rnd, pool, a.max_agentnet) print("agentnet blocks", f, len(g), flush=True) blocchi += g if a.v1_jsonl: g = v1_blocchi(a.v1_jsonl) print("v1 blocks", len(g), flush=True) blocchi += g pool.close() scrivi(blocchi, a.out, rnd) if __name__ == "__main__": main()