File size: 5,513 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | #!/usr/bin/env python3
"""ANALYSE VOCABULAIRE & SEGMENTATION sur les DEUX langues (le §5 n'avait analysé que le lin).
Cherche des défauts SYSTÉMATIQUES corrigeables :
(1) OOV : mots que nous produisons et qui n'existent pas dans le vocabulaire du train
(2) SEGMENTATION : mots collés/séparés à tort (crucial en shona, agglutinant)
(3) substitutions les plus coûteuses, et si elles ont un motif (préfixe, accord, emprunt)
(4) mots du train jamais produits par nous (trous de vocabulaire)
Sur devhard (on a les références) => diagnostic exact, aucune soumission.
"""
import json, os, re
from collections import Counter, defaultdict
import difflib
AUD = "/root/devhard_audio"
def words(t):
return t.split()
def norm(w):
return re.sub(r"[^\w'ɛɔ]", "", w.lower())
def main():
rows = [json.loads(l) for l in open("/root/devhard/devhard_linsna.jsonl", encoding="utf-8")]
D = json.load(open("/root/devhard_allhyps.json", encoding="utf-8"))
MODEL = {"lin": "joint_cont_best", "sna": "sna_ps_best"}
for lg in ("lin", "sna"):
sub = [r for r in rows if r["lang"] == lg]
refs = [r["text"] for r in sub]
H = D[MODEL[lg]]
hyps = [H.get(r["id"], "") for r in sub]
print("\n" + "=" * 78)
print("### %s — %d clips (modele %s)" % (lg.upper(), len(sub), MODEL[lg]))
print("=" * 78)
# vocabulaire du TRAIN (la spec)
tv = Counter()
for l in open("/root/devhard/train_%s_min.jsonl" % lg, encoding="utf-8"):
tv.update(norm(w) for w in json.loads(l).get("text", "").split())
tv.pop("", None)
print("vocabulaire train : %d types, %d tokens" % (len(tv), sum(tv.values())))
# ---------- (1) OOV de nos sorties ----------
hv = Counter()
for h in hyps:
hv.update(norm(w) for w in words(h))
hv.pop("", None)
oov = {w: n for w, n in hv.items() if w not in tv}
ntok = sum(hv.values())
noov = sum(oov.values())
print("\n(1) OOV : %d tokens sur %d (%.1f%%) ; %d types inconnus"
% (noov, ntok, 100.0 * noov / max(ntok, 1), len(oov)))
# OOV des references, pour comparaison (les refs ont aussi des OOV legitimes)
rv = Counter()
for t in refs:
rv.update(norm(w) for w in t.split())
rv.pop("", None)
roov = sum(n for w, n in rv.items() if w not in tv)
print(" (references : %.1f%% d'OOV — c'est la borne naturelle)"
% (100.0 * roov / max(sum(rv.values()), 1)))
print(" top OOV produits :", [w for w, _ in Counter(oov).most_common(12)])
# ---------- (2) segmentation ----------
split_err = Counter() # nous separons ce que la ref colle
join_err = Counter() # nous collons ce que la ref separe
sub_err = Counter()
del_err = Counter()
ins_err = Counter()
for r, h in zip(refs, hyps):
a = [norm(x) for x in r.split()]
b = [norm(x) for x in h.split()]
sm = difflib.SequenceMatcher(None, a, b, autojunk=False)
for op, i1, i2, j1, j2 in sm.get_opcodes():
if op == "replace":
ra, rb = a[i1:i2], b[j1:j2]
if len(ra) == 1 and len(rb) == 2 and ra[0] == rb[0] + rb[1]:
split_err[ra[0]] += 1 # ref collee, nous separons
elif len(ra) == 2 and len(rb) == 1 and rb[0] == ra[0] + ra[1]:
join_err[" ".join(ra)] += 1 # ref separee, nous collons
elif len(ra) == 1 and len(rb) == 1:
sub_err[(ra[0], rb[0])] += 1
elif op == "delete":
for w in a[i1:i2]:
del_err[w] += 1
elif op == "insert":
for w in b[j1:j2]:
ins_err[w] += 1
tot_err = sum(sub_err.values()) + sum(del_err.values()) + sum(ins_err.values()) \
+ sum(split_err.values()) + sum(join_err.values())
print("\n(2) SEGMENTATION :")
print(" ref COLLEE / nous separons : %d cas %s"
% (sum(split_err.values()), split_err.most_common(6)))
print(" ref SEPAREE / nous collons : %d cas %s"
% (sum(join_err.values()), join_err.most_common(6)))
print(" => %.1f%% du budget d'erreur total (%d)"
% (100.0 * (sum(split_err.values()) + sum(join_err.values())) / max(tot_err, 1), tot_err))
# ---------- (3) substitutions systematiques ----------
print("\n(3) TOP substitutions (ref -> nous) :")
for (x, y), n in sub_err.most_common(12):
# meme mot a une lettre pres ?
close = "~" if difflib.SequenceMatcher(None, x, y).ratio() > 0.75 else " "
print(" %s %-18s -> %-18s x%d" % (close, x, y, n))
near = sum(n for (x, y), n in sub_err.items()
if difflib.SequenceMatcher(None, x, y).ratio() > 0.75)
print(" dont %.0f%% sont des QUASI-MOTS (ratio>0.75) = erreurs orthographiques,"
% (100.0 * near / max(sum(sub_err.values()), 1)))
print(" le reste = vrais mots differents (erreurs acoustiques).")
print("\n(4) omissions / insertions les plus frequentes :")
print(" omis :", del_err.most_common(8))
print(" inseres:", ins_err.most_common(8))
print("\nVOCAB_ANALYSIS_DONE")
if __name__ == "__main__":
main()
|