File size: 5,439 Bytes
a882958 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | #!/usr/bin/env python3
"""Aday satırı biçimleri (m2w_filter.py çıktısı üstünde, hızlı CPU işi).
long : eski biçimin uzun sürümü -> etiket | role | type | öznitelikler (<=120) | iç metin (<=200), toplam <=400 karakter
tree : erişilebilirlik-ağacı tarzı -> rol "isim" [durum]
Kullanım: python m2w_format.py --data m2w_v3 --fmt tree --out m2w_v3_tree
"""
import argparse, gzip, json, os
from collections import Counter
LABEL_ATTRS = ("aria_label", "placeholder", "title", "name", "alt", "value", "id")
ROLE_OK = {"button", "link", "checkbox", "radio", "textbox", "searchbox", "combobox", "menuitem", "tab", "option", "switch",
"listbox", "slider", "spinbutton", "menuitemcheckbox", "menuitemradio", "treeitem", "heading", "img", "listitem",
"cell", "row", "columnheader", "menu", "navigation", "dialog"}
INPUT_ROLE = {"checkbox": "checkbox", "radio": "radio", "submit": "button", "button": "button", "reset": "button",
"image": "button", "search": "searchbox", "range": "slider", "number": "spinbutton"}
TAG_ROLE = {"a": "link", "button": "button", "select": "combobox", "textarea": "textbox", "option": "option",
"label": "label", "summary": "button", "li": "listitem", "img": "img", "svg": "img", "i": "img",
"td": "cell", "tr": "row", "th": "columnheader", "h1": "heading", "h2": "heading", "h3": "heading",
"h4": "heading", "p": "paragraph", "span": "generic", "div": "generic"}
def g(at, *keys):
for k in keys:
v = at.get(k)
if v is not None and str(v).strip():
return str(v).strip()
return ""
def fmt_long(c):
at = c.get("at", {})
parts = [c["tag"]]
for k in ("role", "type"):
if at.get(k):
parts.append(str(at[k]))
for k in LABEL_ATTRS:
v = g(at, k, k.replace("_", "-"))
if v:
parts.append(v[:120])
if c.get("inner"):
parts.append(c["inner"][:200])
return " | ".join(parts)[:400]
def role_of(c):
at, tag = c.get("at", {}), c["tag"]
r = g(at, "role").lower()
if r in ROLE_OK:
return r
if tag == "input":
t = g(at, "type").lower()
return INPUT_ROLE.get(t, "textbox")
return TAG_ROLE.get(tag, tag)
def is_true(at, *keys):
for k in keys:
if k in at:
return str(at[k]).strip().lower() not in ("false", "0", "no")
return False
def fmt_tree(c):
at, role = c.get("at", {}), role_of(c)
inner = " ".join(c.get("inner", "").split())
if role in ("textbox", "searchbox", "combobox", "spinbutton"):
order = [g(at, "aria_label", "aria-label"), g(at, "placeholder"), g(at, "title"), g(at, "name"), inner, g(at, "id")]
elif role in ("img",):
order = [g(at, "alt"), g(at, "aria_label", "aria-label"), g(at, "title"), inner, g(at, "name")]
elif role in ("button", "link", "menuitem", "tab", "option", "checkbox", "radio", "switch", "label", "listitem", "heading",
"cell", "row", "columnheader", "paragraph", "generic", "treeitem"):
order = [g(at, "aria_label", "aria-label"), inner, g(at, "value"), g(at, "title"), g(at, "alt"), g(at, "placeholder"),
g(at, "name"), g(at, "id")]
else:
order = [g(at, "aria_label", "aria-label"), inner, g(at, "title"), g(at, "alt"), g(at, "value"), g(at, "name"), g(at, "id")]
name = next((x for x in order if x), "")[:100]
st = []
if is_true(at, "checked", "aria_checked", "aria-checked"):
st.append("checked")
if is_true(at, "selected", "aria_selected", "aria-selected"):
st.append("selected")
if is_true(at, "disabled", "aria_disabled", "aria-disabled"):
st.append("disabled")
for k in ("aria_expanded", "aria-expanded"):
if k in at:
st.append(f"expanded={str(at[k]).strip().lower()}")
break
if is_true(at, "required"):
st.append("required")
s = role + (f' "{name}"' if name else "")
return s + "".join(f" [{x}]" for x in st)
FMT = {"long": fmt_long, "tree": fmt_tree}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--data", required=True); ap.add_argument("--out", required=True)
ap.add_argument("--fmt", required=True, choices=list(FMT))
ap.add_argument("--splits", nargs="+", default=["train", "val", "test_task", "test_website", "test_domain"])
a = ap.parse_args(); os.makedirs(a.out, exist_ok=True); f = FMT[a.fmt]
for s in a.splits:
p = os.path.join(a.data, f"{s}.jsonl.gz")
if not os.path.exists(p):
continue
n = 0; lens = []; roles = Counter()
with gzip.open(p, "rt", encoding="utf-8") as fi, gzip.open(os.path.join(a.out, f"{s}.jsonl.gz"), "wt", encoding="utf-8") as fo:
for l in fi:
ex = json.loads(l)
for c in ex["cands"] + ex.get("dropped_pos", []):
c["text"] = f(c)
lens += [len(c["text"]) for c in ex["cands"]]
if n < 300:
roles.update(c["text"].split(" ")[0] for c in ex["cands"])
fo.write(json.dumps(ex, ensure_ascii=False) + "\n"); n += 1
print(f" {s}: {n} adım | aday metni uzunluğu (karakter) ortanca {sorted(lens)[len(lens)//2]}, p90 {sorted(lens)[int(len(lens)*.9)]}")
print("örnek roller (ilk 300 adım):", roles.most_common(12))
if __name__ == "__main__":
main()
|