Download training/make_data_v2.py from KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA/resolve/main/training/make_data_v2.py
- Command line
-
hf download hf://KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA/training/make_data_v2.py
-
curl -L -o make_data_v2.py https://huggingface.co/KeytechSRL/TernaryPilot-v2-Computer-Browser-Use-LoRA/resolve/main/training/make_data_v2.py
11.4 kB
| """make_data_v2.py: TernaryPilot v2 training data, same format and prompts as v1 (make_data.py), more and different sources: | |
| - GUI-360 (vyokky/GUI-360, MIT): Word/Excel/PowerPoint on Windows. Grounding (agent thought -> point) and action | |
| prediction (task + history + screen -> thought + action), joined by screenshot. API actions (19%) are skipped. | |
| - AgentNet (xlangai/AgentNet, MIT): every completed task with alignment >= 8, Windows AND macOS (v1: Windows, part 1 only). | |
| - Multimodal-Mind2Web (OpenRAIL) and v1's click pool: through make_data.py (--v1-jsonl merges an existing v1 file). | |
| Coordinates are integers on a 0-1000 grid over the image, as in v1. | |
| python make_data_v2.py --gui360 DIR --agentnet-jsonl F --agentnet-images DIR [--v1-jsonl data/train.jsonl] \ | |
| --out data/train_v2.jsonl --max-gui360 40000 --max-agentnet 20000 [--exclude "bench/**/*.png"] | |
| """ | |
| import argparse | |
| import glob | |
| import json | |
| import multiprocessing | |
| import os | |
| import random | |
| import re | |
| from PIL import Image | |
| import make_data as md | |
| _V = [] # dhash of the benchmark images, set before the filtering pool forks | |
| def _hash(path): | |
| return md.dhash(Image.open(path)) | |
| def _libera(path): # the image is not a near copy of any benchmark image | |
| h = _hash(path) | |
| return not any(bin(h ^ v).count("1") <= 12 for v in _V) | |
| def filtra(cands, path, max_n, pool): | |
| """cands already shuffled: the first max_n whose image is free, checked in parallel a slice at a time""" | |
| out = [] | |
| for i in range(0, len(cands), 2048): | |
| parte = cands[i:i + 2048] | |
| out += [c for c, ok in zip(parte, pool.map(_libera, [path(c) for c in parte], chunksize=16)) if ok] | |
| if len(out) >= max_n: | |
| break | |
| return out[:max_n] | |
| def norm(x, y, w, h): | |
| return round(min(max(x / w, 0), 0.999) * 1000), round(min(max(y / h, 0), 0.999) * 1000) | |
| def gui360_azione(js, w, h): | |
| """GUI-360 <tool_call>{function, args} -> our action text, or None (API actions and unknown ones are skipped)""" | |
| f, a = js.get("function"), js.get("args") or {} | |
| for k in ("coordinate", "start_coordinate", "end_coordinate"): # some steps carry null coordinates: skip them | |
| c = a.get(k) | |
| if c is not None and not (isinstance(c, (list, tuple)) and len(c) >= 2 | |
| and all(isinstance(v, (int, float)) for v in c[:2])): | |
| return None | |
| def p(c): | |
| return "x=%d, y=%d" % norm(c[0], c[1], w, h) | |
| if f == "click" and a.get("coordinate"): | |
| b = a.get("button", "left") | |
| nome = "right_click" if b == "right" else "double_click" if a.get("double") else "click" | |
| return f"{nome}({p(a['coordinate'])})" | |
| if f == "type": | |
| testo = a.get("text", a.get("keys", "")) | |
| if not isinstance(testo, str): | |
| return None | |
| keys = re.fullmatch(r"\{VK_(\w+)\}(\w)", testo) | |
| pre = f"click({p(a['coordinate'])})\n" if a.get("coordinate") else "" | |
| if keys: | |
| return pre + 'hotkey("%s+%s")' % (keys.group(1).lower().replace("control", "ctrl"), keys.group(2).lower()) | |
| if re.fullmatch(r"\{(ENTER|TAB|ESC|DELETE|BACKSPACE)\}", testo): | |
| return pre + 'press("%s")' % testo.strip("{}").lower() | |
| return pre + "write(%s)" % json.dumps(testo, ensure_ascii=False) | |
| if f == "drag" and a.get("start_coordinate") and a.get("end_coordinate"): | |
| return f"move({p(a['start_coordinate'])})\ndrag({p(a['end_coordinate'])})" | |
| if f == "wheel_mouse_input" and a.get("coordinate") is not None: | |
| return f"move({p(a['coordinate'])})\nscroll({int(a.get('wheel_dist', 0))})" | |
| return None | |
| def gui360(cartella, rnd, pool, max_n): | |
| """blocks of examples from processed_data/{grounding_resize, action_prediction_train_resize}""" | |
| gr = json.load(open(f"{cartella}/processed_data/grounding_resize/training_data.json", encoding="utf-8")) | |
| ap = json.load(open(f"{cartella}/processed_data/action_prediction_train_resize/training_data.json", encoding="utf-8")) | |
| pensiero, punto = {}, {} | |
| for r in gr: # thought -> point, keyed by screenshot | |
| img = r["images"][0].replace("\\", "/") | |
| m = re.search(r"The instruction is:\n(.*?)\n\nOutput the coordinate", r["conversation"][0]["value"], re.S) | |
| c = re.search(r"\[(\d+),\s*(\d+)\]", r["conversation"][1]["value"]) | |
| if m and c: | |
| pensiero[img] = md.frase(m.group(1), 300) | |
| punto[img] = (int(c.group(1)), int(c.group(2))) | |
| traiett = {} | |
| for r in ap: # group steps by trajectory (id = <...>_<traj>_<step>) | |
| img = r["images"][0].replace("\\", "/") | |
| tid, step = r["id"].rsplit("_", 1) | |
| q = re.search(r"The instruction is:\n(.*?)\n\nThe history", r["conversation"][0]["value"], re.S) | |
| t = re.search(r"<tool_call>\s*(\{.*\})\s*</tool_call>", r["conversation"][1]["value"], re.S) | |
| if q and t: | |
| try: | |
| traiett.setdefault(tid, []).append((int(step), img, q.group(1).strip(), json.loads(t.group(1)))) | |
| except ValueError: | |
| pass | |
| cands = [] | |
| radice = f"{cartella}/processed_data/action_prediction_train_resize" | |
| gradice = f"{cartella}/processed_data/grounding_resize" | |
| for tid, passi in traiett.items(): | |
| storia = [] | |
| for step, img, compito, js in sorted(passi): | |
| path = os.path.join(radice, img) | |
| if not os.path.exists(path): | |
| continue | |
| w, h = Image.open(path).size # header only | |
| act = gui360_azione(js, w, h) | |
| th = pensiero.get(img) | |
| if act: | |
| if th: | |
| gp = os.path.join(gradice, img) | |
| gp = gp if img in punto and os.path.exists(gp) else None | |
| cands.append((path, compito, list(storia), act, th, gp, punto.get(img))) | |
| storia.append(md.frase(th or act.splitlines()[0], 160)) | |
| rnd.shuffle(cands) | |
| print("gui360 candidates", len(cands), flush=True) | |
| blocchi = [] | |
| for path, compito, storia, act, th, gp, pt in filtra(cands, lambda c: c[0], max_n, pool): | |
| b = [{"cache": md.md5(path), "img": path, "user": md.p_azione("Windows computer", compito, storia), | |
| "risposta": f"Thought: {th}\nAction: {act}", "tipo": "action_gui360"}] | |
| if gp: | |
| gw, gh = Image.open(gp).size | |
| x, y = norm(*pt, gw, gh) | |
| b.append({"cache": md.md5(gp), "img": gp, "user": md.P_CLIC + th, "risposta": '{"x": %d, "y": %d}' % (x, y), | |
| "tipo": "click_gui360"}) | |
| blocchi.append(b) | |
| return blocchi | |
| def agentnet(jsonl, cartella, rnd, pool, max_n): | |
| mac = re.compile(r"\bmac ?os\b|\bfinder\b|\bdock\b|\bmenu bar\b|\bcommand key|\bcmd\b|safari|spotlight", re.I) | |
| passi = [] | |
| for riga in open(jsonl, encoding="utf-8"): | |
| d = json.loads(riga) | |
| if not d.get("task_completed") or (d.get("alignment_score") or 0) < 8: | |
| continue | |
| oss = " ".join(s["value"].get("observation", "") for s in d["traj"][:3]) | |
| dove = "Mac computer" if mac.search(oss) else "Windows computer" | |
| compito, storia = d.get("natural_language_task") or d["instruction"], [] | |
| for s in d["traj"]: | |
| v, img = s["value"], os.path.join(cartella, s["image"]) | |
| act = md.converti(v.get("code") or "") | |
| if act and os.path.exists(img) and v.get("last_step_correct", True) is not False: | |
| passi.append((img, dove, compito, list(storia), v, act)) | |
| storia.append(md.frase(v.get("action", ""), 160)) | |
| rnd.shuffle(passi) | |
| blocchi = [] | |
| for img, dove, compito, storia, v, act in filtra(passi, lambda p: p[0], max_n, pool): | |
| k = md.md5(img) | |
| b = [{"cache": k, "img": img, "user": md.p_azione(dove, compito, storia), | |
| "risposta": f"Thought: {md.frase(v.get('action', ''), 220)}\nAction: {act}", "tipo": "action_pc"}] | |
| if rnd.random() < 0.25 and v.get("observation"): | |
| b.append({"cache": k, "img": img, "user": md.P_LETTURA, "risposta": md.frase(v["observation"], 900), "tipo": "reading"}) | |
| mm = re.fullmatch(r"(?:double_|right_)?click\((x=\d+, y=\d+)\)", act) | |
| t = re.sub(r"^(double[- ]|right[- ])?click (on |the )*", "", md.frase(v.get("action", ""), 120), flags=re.I).rstrip(".") | |
| if mm and 3 <= len(t) <= 100: | |
| x, y = re.findall(r"\d+", mm.group(1)) | |
| b.append({"cache": k, "img": img, "user": md.P_CLIC + t, "risposta": '{"x": %s, "y": %s}' % (x, y), "tipo": "click_pc"}) | |
| blocchi.append(b) | |
| return blocchi | |
| def v1_blocchi(path): # an existing v1 train.jsonl, regrouped by screenshot | |
| per = {} | |
| for riga in open(path, encoding="utf-8"): | |
| e = json.loads(riga) | |
| per.setdefault(e["cache"], []).append(e) | |
| return list(per.values()) | |
| def scrivi(blocchi, out, rnd): # same grouping as v1: blocks of one screenshot stay inside one window of 4 | |
| rnd.shuffle(blocchi) | |
| finestre, aperte = [], [] | |
| for b in blocchi: | |
| b = b[:4] | |
| for f in aperte: | |
| if sum(map(len, f)) + len(b) <= 4: | |
| f.append(b) | |
| break | |
| else: | |
| aperte.append([b]) | |
| for f in [f for f in aperte if sum(map(len, f)) == 4]: | |
| finestre.append(f) | |
| aperte.remove(f) | |
| finestre += aperte | |
| rnd.shuffle(finestre) | |
| os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True) | |
| tipi = {} | |
| with open(out, "w", encoding="utf-8") as fo: | |
| for f in finestre: | |
| for b in f: | |
| for e in b: | |
| fo.write(json.dumps(e, ensure_ascii=False) + "\n") | |
| tipi[e["tipo"]] = tipi.get(e["tipo"], 0) + 1 | |
| print("examples", sum(tipi.values()), "screenshots", len(blocchi), tipi) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--gui360"), ap.add_argument("--agentnet-jsonl", nargs="*"), ap.add_argument("--agentnet-images") | |
| ap.add_argument("--v1-jsonl"), ap.add_argument("--exclude"), ap.add_argument("--out", required=True) | |
| ap.add_argument("--max-gui360", type=int, default=40000), ap.add_argument("--max-agentnet", type=int, default=20000) | |
| ap.add_argument("--seed", type=int, default=0) | |
| a = ap.parse_args() | |
| rnd = random.Random(a.seed) | |
| n = max(1, (os.cpu_count() or 2) - 1) | |
| if a.exclude: | |
| with multiprocessing.Pool(n) as p: | |
| _V[:] = p.map(_hash, glob.glob(a.exclude, recursive=True), chunksize=8) | |
| print("excluded benchmark images", len(_V), flush=True) | |
| pool = multiprocessing.Pool(n) # forked after _V is set, so the workers see it | |
| blocchi = [] | |
| if a.gui360: | |
| g = gui360(a.gui360, rnd, pool, a.max_gui360) | |
| print("gui360 blocks", len(g), flush=True) | |
| blocchi += g | |
| for f in a.agentnet_jsonl or []: | |
| g = agentnet(f, a.agentnet_images, rnd, pool, a.max_agentnet) | |
| print("agentnet blocks", f, len(g), flush=True) | |
| blocchi += g | |
| if a.v1_jsonl: | |
| g = v1_blocchi(a.v1_jsonl) | |
| print("v1 blocks", len(g), flush=True) | |
| blocchi += g | |
| pool.close() | |
| scrivi(blocchi, a.out, rnd) | |
| if __name__ == "__main__": | |
| main() | |