archive/scripts/audit_STATSLING3.py

# -*- coding: utf-8 -*-
"""Audit 3: mecanisme de suppression OOV/virgules par le KenLM mot ; sweep JOIN ;
informativite de la decision point-final."""
import csv, json, re, sys
from collections import Counter

OUT = []
def P(*a):
    s = " ".join(str(x) for x in a); OUT.append(s); print(s)

sub = {}
with open("/root/sub_LMA06.csv", encoding="utf-8") as f:
    for row in csv.DictReader(f): sub[row["ID"]] = row["Target"]
lang = json.load(open("/root/test_lang.json", encoding="utf-8"))
hyp = {"lin": [], "sna": []}
for k, v in sub.items():
    if lang.get(k) in hyp: hyp[lang[k]].append(v)
train, val = {"lin": [], "sna": []}, {"lin": [], "sna": []}
for L in ("lin", "sna"):
    for sp, st in (("train", train), ("validation", val)):
        with open("/scratch/prep/manifests/waxal_%s_%s.jsonl" % (L, sp), encoding="utf-8") as f:
            for line in f:
                if line.strip(): st[L].append(json.loads(line)["text"])

tot_w = sum(len(t.split()) for L in hyp for t in hyp[L])
tot_c = sum(len(t) for L in hyp for t in hyp[L])
W_ERR, C_ERR = 0.5/tot_w, 0.5/tot_c

# ===== 1. VOCABULAIRE DU KenLM : que voit vraiment le decodeur ? =====
P("=== 1. VOCABULAIRE DES ARPA (unigrammes = ce que pyctcdecode considere 'connu') ===")
for L, arpa in (("lin", "/scratch/lm/lin_5g.arpa"), ("sna", "/scratch/lm/sna_5g.arpa")):
    uni = set(); state = 0
    with open(arpa, encoding="utf-8", errors="replace") as f:
        for line in f:
            line = line.rstrip("\n")
            if line.startswith("\\1-grams:"): state = 1; continue
            if line.startswith("\\2-grams:"): break
            if state == 1 and line.strip():
                parts = line.split("\t")
                if len(parts) >= 2: uni.add(parts[1])
    withcomma = [w for w in uni if w.endswith(",")]
    withdot = [w for w in uni if w.endswith(".")]
    P("%s | ARPA unigrammes=%d | finissant par ',' = %d (%.2f%%) | par '.' = %d (%.2f%%)"
      % (L, len(uni), len(withcomma), 100.0*len(withcomma)/len(uni), len(withdot), 100.0*len(withdot)/len(uni)))
    # part des tokens du corpus LM portant une virgule
    corpus = "/scratch/lm/corpus_%s.txt" % L
    n = nc = 0
    with open(corpus, encoding="utf-8") as f:
        for line in f:
            ws = line.split(); n += len(ws); nc += sum(1 for w in ws if w.endswith(","))
    P("%s | corpus LM: %d tokens, %d portent une virgule finale (%.2f%%)" % (L, n, nc, 100.0*nc/n))
    # combien de formes 'mot,' sont dans l'ARPA vs formes 'mot' nues
    bare = set(w.rstrip(",.") for w in withcomma)
    P("%s | %d formes-virgule distinctes ; %d ont AUSSI la forme nue dans l'ARPA"
      % (L, len(withcomma), sum(1 for b in bare if b in uni)))
    # taux OOV des references val vis a vis de l'ARPA (ce que le decodeur ne peut pas produire sans penalite)
    tw = ow = 0
    for t in val[L]:
        for w in t.split():
            tw += 1
            if w not in uni: ow += 1
    P("%s | tokens de reference VAL absents de l'ARPA = %.2f%% -> chacun paie unk_score_offset=-10" % (L, 100.0*ow/tw))

# ===== 2. SWEEP JOIN : precision/couverture mesuree en held-out sur VAL =====
P("")
P("=== 2. SWEEP DES REGLES JOIN (precision held-out mesuree sur VAL) ===")
PUNCT = ".,;:!?\"()«»"
def toks(t): return [x for x in (w.strip(PUNCT) for w in t.split()) if x]
for L in ("lin", "sna"):
    uni = Counter(); bi = Counter()
    for t in train[L]:
        ws = [w.lower() for w in toks(t)]; uni.update(ws); bi.update(zip(ws, ws[1:]))
    uni_v = Counter(); bi_v = Counter()
    for t in val[L]:
        ws = [w.lower() for w in toks(t)]; uni_v.update(ws); bi_v.update(zip(ws, ws[1:]))
    hyp_bi = Counter()
    nhw = 0
    for t in hyp[L]:
        ws = [w.lower() for w in toks(t)]; nhw += len(ws); hyp_bi.update(zip(ws, ws[1:]))
    P("--- %s (mots hyp=%d) ---" % (L, nhw))
    P("   %-8s %-8s %8s %10s %10s %8s %10s" % ("minJoin", "maxRatio", "regles", "declench.", "valTP", "valFP", "prec%"))
    for minj in (5, 10, 20, 50):
        for mr in (0.02, 0.05, 0.10, 0.25):
            rules = {}
            for j, cj in uni.items():
                if cj < minj: continue
                for i in range(2, len(j)-1):
                    a, b = j[:i], j[i:]
                    if uni.get(a,0) < 20 or uni.get(b,0) < 20: continue
                    cb = bi.get((a,b), 0)
                    if cb <= mr*cj: rules[(a,b)] = (cj, cb)
            trig = sum(c for pair, c in hyp_bi.items() if pair in rules)
            tp = sum(uni_v.get(a+b,0) for (a,b) in rules)
            fp = sum(bi_v.get((a,b),0) for (a,b) in rules)
            prec = tp/max(tp+fp,1)
            P("   %-8d %-8.2f %8d %10d %10d %10d %8.2f" % (minj, mr, len(rules), trig, tp, fp, 100*prec))
    # meilleure config detaillee
    rules = {}
    for j, cj in uni.items():
        if cj < 10: continue
        for i in range(2, len(j)-1):
            a, b = j[:i], j[i:]
            if uni.get(a,0) < 20 or uni.get(b,0) < 20: continue
            cb = bi.get((a,b), 0)
            if cb <= 0.05*cj: rules[(a,b)] = (cj, cb)
    trigs = Counter({p: c for p, c in hyp_bi.items() if p in rules})
    tp = sum(uni_v.get(a+b,0) for (a,b) in rules); fp = sum(bi_v.get((a,b),0) for (a,b) in rules)
    prec = tp/max(tp+fp,1); n = sum(trigs.values())
    P("   [minJoin=10, maxRatio=0.05] declenchements: %s" % ", ".join("%s+%s x%d" % (a,b,c) for (a,b),c in trigs.most_common(20)))
    g = n*(2*prec-1)*2
    P("   GAIN = %d decl. x 2(2p-1)=%.2f -> %.1f err-mot = %+.5f score" % (n, 2*(2*prec-1), g, g*W_ERR))

# ===== 3. INFORMATIVITE DE LA DECISION POINT-FINAL =====
P("")
P("=== 3. NOTRE DECISION 'point final' PORTE-T-ELLE DE L'INFORMATION ? (devhard) ===")
try:
    allh = json.load(open("/root/devhard_allhyps.json", encoding="utf-8"))
    refs = {}
    import os
    for fn in os.listdir("/root/devhard"):
        with open("/root/devhard/"+fn, encoding="utf-8") as f:
            for line in f:
                if line.strip():
                    d = json.loads(line); refs[d["id"]] = d.get("text","")
    P("devhard: %d references chargees ; modeles dispo: %s" % (len(refs), list(allh.keys())[:6]))
    for mdl, hyps in allh.items():
        pairs = [(h.strip().endswith("."), refs[i].strip().endswith(".")) for i, h in hyps.items() if i in refs and refs[i].strip()]
        if len(pairs) < 50: continue
        n = len(pairs)
        ph = sum(1 for a,b in pairs if a)/n; pr = sum(1 for a,b in pairs if b)/n
        acc_now = sum(1 for a,b in pairs if a==b)/n
        acc_all = pr
        # correlation phi
        n11 = sum(1 for a,b in pairs if a and b); n10 = sum(1 for a,b in pairs if a and not b)
        n01 = sum(1 for a,b in pairs if not a and b); n00 = sum(1 for a,b in pairs if not a and not b)
        den = ((n11+n10)*(n01+n00)*(n11+n01)*(n10+n00))**0.5
        phi = ((n11*n00 - n10*n01)/den) if den > 0 else 0.0
        P("%-22s n=%4d | notre taux '.'=%.3f ref=%.3f | exact actuel=%.3f  si TOUJOURS='.'=%.3f | phi=%+.3f"
          % (mdl, n, ph, pr, acc_now, acc_all, phi))
except Exception as e:
    P("devhard indisponible: %r" % (e,))

# ===== 4. OOV : combien de tokens de reference nous sont structurellement inatteignables =====
P("")
P("=== 4. PLAFOND STRUCTUREL OOV (lingala) ===")
for L in ("lin", "sna"):
    tr = Counter()
    for t in train[L]: tr.update(w.lower() for w in toks(t))
    vw = [w.lower() for t in val[L] for w in toks(t)]
    nat = sum(1 for w in vw if w not in tr)/len(vw)
    hw = [w.lower() for t in hyp[L] for w in toks(t)]
    ours = sum(1 for w in hw if w not in tr)/len(hw)
    deficit = (nat - ours)*len(hw)
    P("%s | OOV naturel(ref)=%.2f%%  notre OOV=%.2f%%  ratio=%.2f" % (L, 100*nat, 100*ours, ours/nat))
    P("%s | tokens de ref probablement nouveaux que nous ne pouvons PAS produire ~= %.0f" % (L, max(deficit,0)))
    P("%s | pool bloque = %.0f err-mot = %.5f score" % (L, max(deficit,0), max(deficit,0)*W_ERR))

with open("/root/audit_STATSLING3_out.txt","w",encoding="utf-8") as f: f.write("\n".join(OUT))
print("\n[OK] /root/audit_STATSLING3_out.txt")