|
|
| """RÈGLES DE CASSE & PONCTUATION FINALE, apprises sur le corpus TRAIN (la spec).
|
| Écarts systématiques mesurés :
|
| LIN : majuscules internes refs 129 vs nous 9 -> on en met 14x trop PEU (noms propres)
|
| SNA : majuscules internes refs 574 vs nous 635 -> on en met trop
|
| SNA : ponctuation finale refs 411/433 vs nous 431/433 -> on en met trop
|
| Casse+ponctuation = 8.5% du budget lin et 24.2% du budget sna.
|
| Règle : pour chaque mot, le train dit-il majuscule ou minuscule en position INTERNE ?
|
| On ne corrige que les cas tranchés (ratio >= R, effectif >= N).
|
| """
|
| import json, os, pickle, re
|
| from collections import Counter
|
| import jiwer
|
| from transformers import AutoProcessor
|
|
|
| R = float(os.environ.get("R", "4"))
|
| N = int(os.environ.get("N", "5"))
|
|
|
|
|
| def sc(refs, hyps):
|
| pr = [(r, h) for r, h in zip(refs, hyps) if r.strip()]
|
| a = [x for x, _ in pr]; b = [y for _, y in pr]
|
| w = jiwer.wer(a, b); c = jiwer.cer(a, b)
|
| return w, c, 0.5 * w + 0.5 * c
|
|
|
|
|
| def key(w):
|
| return re.sub(r"[^\w'ɛɔ]", "", w).lower()
|
|
|
|
|
| rows = [json.loads(l) for l in open("/root/devhard/devhard_linsna.jsonl", encoding="utf-8")]
|
| CFG = {"lin": ("/root/models/joint_cont_best", "/scratch/lm/logits_lin.pkl"),
|
| "sna": ("/root/models/sna_ps_best", "/scratch/lm/logits_sna.pkl")}
|
|
|
| for lg, (mdl, lgt) in CFG.items():
|
| sub = [r for r in rows if r["lang"] == lg]
|
| refs = [r["text"] for r in sub]
|
| L1 = pickle.load(open(lgt, "rb"))
|
| tok = AutoProcessor.from_pretrained(mdl).tokenizer
|
| hyps = [" ".join(tok.decode(l.argmax(-1)).replace("|", " ").split()) for l in L1]
|
| base = sc(refs, hyps)
|
| print("\n=== %s : baseline combine %.4f ===" % (lg.upper(), base[2]))
|
|
|
|
|
| up = Counter(); lo = Counter(); endp = 0; ntot = 0
|
| for l in open("/root/devhard/train_%s_min.jsonl" % lg, encoding="utf-8"):
|
| t = json.loads(l).get("text", "").strip()
|
| if not t:
|
| continue
|
| ntot += 1
|
| if t.rstrip()[-1:] in ".!?":
|
| endp += 1
|
| ws = t.split()
|
| for i, w in enumerate(ws):
|
| if i == 0 or ws[i - 1].rstrip()[-1:] in ".!?":
|
| continue
|
| k = key(w)
|
| if not k:
|
| continue
|
| (up if w[:1].isupper() else lo)[k] += 1
|
| print(" train : %.1f%% des phrases finissent par une ponctuation" % (100.0 * endp / max(ntot, 1)))
|
|
|
| force_up = {k for k in up if up[k] >= N and up[k] >= R * lo.get(k, 0)}
|
| force_lo = {k for k in lo if lo[k] >= N and lo[k] >= R * up.get(k, 0)}
|
| print(" mots TOUJOURS majuscule en interne : %d | TOUJOURS minuscule : %d"
|
| % (len(force_up), len(force_lo)))
|
|
|
| def fix_case(t):
|
| ws = t.split()
|
| out = [ws[0]] if ws else []
|
| for i, w in enumerate(ws[1:], 1):
|
| if ws[i - 1].rstrip()[-1:] in ".!?":
|
| out.append(w); continue
|
| k = key(w)
|
| if k in force_up and not w[:1].isupper():
|
| w = w[:1].upper() + w[1:]
|
| elif k in force_lo and w[:1].isupper():
|
| w = w[:1].lower() + w[1:]
|
| out.append(w)
|
| return " ".join(out)
|
|
|
| def fix_endp(t, keep):
|
| """si le train met rarement une ponctuation finale, on la retire"""
|
| if keep:
|
| return t
|
| return t.rstrip(".!?").rstrip() if t.rstrip()[-1:] in ".!?" else t
|
|
|
| keep_final = (endp / max(ntot, 1)) >= 0.5
|
| for lbl, fn in (("casse seule", lambda t: fix_case(t)),
|
| ("ponct finale seule", lambda t: fix_endp(t, keep_final)),
|
| ("les deux", lambda t: fix_endp(fix_case(t), keep_final))):
|
| o = [fn(h) for h in hyps]
|
| m = sc(refs, o)
|
| nch = sum(1 for a, b in zip(hyps, o) if a != b)
|
| print(" %-20s WER %.4f CER %.4f combine %.4f (%+.4f) %d clips%s"
|
| % (lbl, m[0], m[1], m[2], m[2] - base[2], nch,
|
| " <-- GAIN" if m[2] < base[2] - 0.0005 else ""))
|
| print("\nCASE_RULES_DONE")
|
|
|
