phase2_corrected/code/vocab_analysis.py

#!/usr/bin/env python3
"""ANALYSE VOCABULAIRE & SEGMENTATION sur les DEUX langues (le §5 n'avait analysé que le lin).
Cherche des défauts SYSTÉMATIQUES corrigeables :
 (1) OOV : mots que nous produisons et qui n'existent pas dans le vocabulaire du train
 (2) SEGMENTATION : mots collés/séparés à tort (crucial en shona, agglutinant)
 (3) substitutions les plus coûteuses, et si elles ont un motif (préfixe, accord, emprunt)
 (4) mots du train jamais produits par nous (trous de vocabulaire)
Sur devhard (on a les références) => diagnostic exact, aucune soumission.
"""
import json, os, re
from collections import Counter, defaultdict
import difflib

AUD = "/root/devhard_audio"


def words(t):
    return t.split()


def norm(w):
    return re.sub(r"[^\w'ɛɔ]", "", w.lower())


def main():
    rows = [json.loads(l) for l in open("/root/devhard/devhard_linsna.jsonl", encoding="utf-8")]
    D = json.load(open("/root/devhard_allhyps.json", encoding="utf-8"))
    MODEL = {"lin": "joint_cont_best", "sna": "sna_ps_best"}

    for lg in ("lin", "sna"):
        sub = [r for r in rows if r["lang"] == lg]
        refs = [r["text"] for r in sub]
        H = D[MODEL[lg]]
        hyps = [H.get(r["id"], "") for r in sub]
        print("\n" + "=" * 78)
        print("### %s — %d clips (modele %s)" % (lg.upper(), len(sub), MODEL[lg]))
        print("=" * 78)

        # vocabulaire du TRAIN (la spec)
        tv = Counter()
        for l in open("/root/devhard/train_%s_min.jsonl" % lg, encoding="utf-8"):
            tv.update(norm(w) for w in json.loads(l).get("text", "").split())
        tv.pop("", None)
        print("vocabulaire train : %d types, %d tokens" % (len(tv), sum(tv.values())))

        # ---------- (1) OOV de nos sorties ----------
        hv = Counter()
        for h in hyps:
            hv.update(norm(w) for w in words(h))
        hv.pop("", None)
        oov = {w: n for w, n in hv.items() if w not in tv}
        ntok = sum(hv.values())
        noov = sum(oov.values())
        print("\n(1) OOV : %d tokens sur %d (%.1f%%) ; %d types inconnus"
              % (noov, ntok, 100.0 * noov / max(ntok, 1), len(oov)))
        # OOV des references, pour comparaison (les refs ont aussi des OOV legitimes)
        rv = Counter()
        for t in refs:
            rv.update(norm(w) for w in t.split())
        rv.pop("", None)
        roov = sum(n for w, n in rv.items() if w not in tv)
        print("    (references : %.1f%% d'OOV — c'est la borne naturelle)"
              % (100.0 * roov / max(sum(rv.values()), 1)))
        print("    top OOV produits :", [w for w, _ in Counter(oov).most_common(12)])

        # ---------- (2) segmentation ----------
        split_err = Counter()   # nous separons ce que la ref colle
        join_err = Counter()    # nous collons ce que la ref separe
        sub_err = Counter()
        del_err = Counter()
        ins_err = Counter()
        for r, h in zip(refs, hyps):
            a = [norm(x) for x in r.split()]
            b = [norm(x) for x in h.split()]
            sm = difflib.SequenceMatcher(None, a, b, autojunk=False)
            for op, i1, i2, j1, j2 in sm.get_opcodes():
                if op == "replace":
                    ra, rb = a[i1:i2], b[j1:j2]
                    if len(ra) == 1 and len(rb) == 2 and ra[0] == rb[0] + rb[1]:
                        split_err[ra[0]] += 1          # ref collee, nous separons
                    elif len(ra) == 2 and len(rb) == 1 and rb[0] == ra[0] + ra[1]:
                        join_err[" ".join(ra)] += 1     # ref separee, nous collons
                    elif len(ra) == 1 and len(rb) == 1:
                        sub_err[(ra[0], rb[0])] += 1
                elif op == "delete":
                    for w in a[i1:i2]:
                        del_err[w] += 1
                elif op == "insert":
                    for w in b[j1:j2]:
                        ins_err[w] += 1
        tot_err = sum(sub_err.values()) + sum(del_err.values()) + sum(ins_err.values()) \
            + sum(split_err.values()) + sum(join_err.values())
        print("\n(2) SEGMENTATION :")
        print("    ref COLLEE / nous separons : %d cas   %s"
              % (sum(split_err.values()), split_err.most_common(6)))
        print("    ref SEPAREE / nous collons : %d cas   %s"
              % (sum(join_err.values()), join_err.most_common(6)))
        print("    => %.1f%% du budget d'erreur total (%d)"
              % (100.0 * (sum(split_err.values()) + sum(join_err.values())) / max(tot_err, 1), tot_err))

        # ---------- (3) substitutions systematiques ----------
        print("\n(3) TOP substitutions (ref -> nous) :")
        for (x, y), n in sub_err.most_common(12):
            # meme mot a une lettre pres ?
            close = "~" if difflib.SequenceMatcher(None, x, y).ratio() > 0.75 else " "
            print("    %s %-18s -> %-18s x%d" % (close, x, y, n))
        near = sum(n for (x, y), n in sub_err.items()
                   if difflib.SequenceMatcher(None, x, y).ratio() > 0.75)
        print("    dont %.0f%% sont des QUASI-MOTS (ratio>0.75) = erreurs orthographiques,"
              % (100.0 * near / max(sum(sub_err.values()), 1)))
        print("    le reste = vrais mots differents (erreurs acoustiques).")

        print("\n(4) omissions / insertions les plus frequentes :")
        print("    omis   :", del_err.most_common(8))
        print("    inseres:", ins_err.most_common(8))

    print("\nVOCAB_ANALYSIS_DONE")


if __name__ == "__main__":
    main()