| |
| """Aday satırı biçimleri (m2w_filter.py çıktısı üstünde, hızlı CPU işi). |
| |
| long : eski biçimin uzun sürümü -> etiket | role | type | öznitelikler (<=120) | iç metin (<=200), toplam <=400 karakter |
| tree : erişilebilirlik-ağacı tarzı -> rol "isim" [durum] |
| |
| Kullanım: python m2w_format.py --data m2w_v3 --fmt tree --out m2w_v3_tree |
| """ |
| import argparse, gzip, json, os |
| from collections import Counter |
|
|
| LABEL_ATTRS = ("aria_label", "placeholder", "title", "name", "alt", "value", "id") |
| ROLE_OK = {"button", "link", "checkbox", "radio", "textbox", "searchbox", "combobox", "menuitem", "tab", "option", "switch", |
| "listbox", "slider", "spinbutton", "menuitemcheckbox", "menuitemradio", "treeitem", "heading", "img", "listitem", |
| "cell", "row", "columnheader", "menu", "navigation", "dialog"} |
| INPUT_ROLE = {"checkbox": "checkbox", "radio": "radio", "submit": "button", "button": "button", "reset": "button", |
| "image": "button", "search": "searchbox", "range": "slider", "number": "spinbutton"} |
| TAG_ROLE = {"a": "link", "button": "button", "select": "combobox", "textarea": "textbox", "option": "option", |
| "label": "label", "summary": "button", "li": "listitem", "img": "img", "svg": "img", "i": "img", |
| "td": "cell", "tr": "row", "th": "columnheader", "h1": "heading", "h2": "heading", "h3": "heading", |
| "h4": "heading", "p": "paragraph", "span": "generic", "div": "generic"} |
|
|
|
|
| def g(at, *keys): |
| for k in keys: |
| v = at.get(k) |
| if v is not None and str(v).strip(): |
| return str(v).strip() |
| return "" |
|
|
|
|
| def fmt_long(c): |
| at = c.get("at", {}) |
| parts = [c["tag"]] |
| for k in ("role", "type"): |
| if at.get(k): |
| parts.append(str(at[k])) |
| for k in LABEL_ATTRS: |
| v = g(at, k, k.replace("_", "-")) |
| if v: |
| parts.append(v[:120]) |
| if c.get("inner"): |
| parts.append(c["inner"][:200]) |
| return " | ".join(parts)[:400] |
|
|
|
|
| def role_of(c): |
| at, tag = c.get("at", {}), c["tag"] |
| r = g(at, "role").lower() |
| if r in ROLE_OK: |
| return r |
| if tag == "input": |
| t = g(at, "type").lower() |
| return INPUT_ROLE.get(t, "textbox") |
| return TAG_ROLE.get(tag, tag) |
|
|
|
|
| def is_true(at, *keys): |
| for k in keys: |
| if k in at: |
| return str(at[k]).strip().lower() not in ("false", "0", "no") |
| return False |
|
|
|
|
| def fmt_tree(c): |
| at, role = c.get("at", {}), role_of(c) |
| inner = " ".join(c.get("inner", "").split()) |
| if role in ("textbox", "searchbox", "combobox", "spinbutton"): |
| order = [g(at, "aria_label", "aria-label"), g(at, "placeholder"), g(at, "title"), g(at, "name"), inner, g(at, "id")] |
| elif role in ("img",): |
| order = [g(at, "alt"), g(at, "aria_label", "aria-label"), g(at, "title"), inner, g(at, "name")] |
| elif role in ("button", "link", "menuitem", "tab", "option", "checkbox", "radio", "switch", "label", "listitem", "heading", |
| "cell", "row", "columnheader", "paragraph", "generic", "treeitem"): |
| order = [g(at, "aria_label", "aria-label"), inner, g(at, "value"), g(at, "title"), g(at, "alt"), g(at, "placeholder"), |
| g(at, "name"), g(at, "id")] |
| else: |
| order = [g(at, "aria_label", "aria-label"), inner, g(at, "title"), g(at, "alt"), g(at, "value"), g(at, "name"), g(at, "id")] |
| name = next((x for x in order if x), "")[:100] |
| st = [] |
| if is_true(at, "checked", "aria_checked", "aria-checked"): |
| st.append("checked") |
| if is_true(at, "selected", "aria_selected", "aria-selected"): |
| st.append("selected") |
| if is_true(at, "disabled", "aria_disabled", "aria-disabled"): |
| st.append("disabled") |
| for k in ("aria_expanded", "aria-expanded"): |
| if k in at: |
| st.append(f"expanded={str(at[k]).strip().lower()}") |
| break |
| if is_true(at, "required"): |
| st.append("required") |
| s = role + (f' "{name}"' if name else "") |
| return s + "".join(f" [{x}]" for x in st) |
|
|
|
|
| FMT = {"long": fmt_long, "tree": fmt_tree} |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--data", required=True); ap.add_argument("--out", required=True) |
| ap.add_argument("--fmt", required=True, choices=list(FMT)) |
| ap.add_argument("--splits", nargs="+", default=["train", "val", "test_task", "test_website", "test_domain"]) |
| a = ap.parse_args(); os.makedirs(a.out, exist_ok=True); f = FMT[a.fmt] |
| for s in a.splits: |
| p = os.path.join(a.data, f"{s}.jsonl.gz") |
| if not os.path.exists(p): |
| continue |
| n = 0; lens = []; roles = Counter() |
| with gzip.open(p, "rt", encoding="utf-8") as fi, gzip.open(os.path.join(a.out, f"{s}.jsonl.gz"), "wt", encoding="utf-8") as fo: |
| for l in fi: |
| ex = json.loads(l) |
| for c in ex["cands"] + ex.get("dropped_pos", []): |
| c["text"] = f(c) |
| lens += [len(c["text"]) for c in ex["cands"]] |
| if n < 300: |
| roles.update(c["text"].split(" ")[0] for c in ex["cands"]) |
| fo.write(json.dumps(ex, ensure_ascii=False) + "\n"); n += 1 |
| print(f" {s}: {n} adım | aday metni uzunluğu (karakter) ortanca {sorted(lens)[len(lens)//2]}, p90 {sorted(lens)[int(len(lens)*.9)]}") |
| print("örnek roller (ilk 300 adım):", roles.most_common(12)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
