# -*- coding: utf-8 -*-
import csv, json
sub = {}
for r in csv.DictReader(open("/root/sub_LMA06.csv", encoding="utf-8")): sub[r["ID"]] = r["Target"]
lang = json.load(open("/root/test_lang.json", encoding="utf-8"))

def arpa_uni(p):
    u = set(); s = 0
    for line in open(p, encoding="utf-8", errors="replace"):
        if line.startswith("\\1-grams:"): s = 1; continue
        if line.startswith("\\2-grams:"): break
        if s and line.strip():
            q = line.split("\t")
            if len(q) >= 2: u.add(q[1])
    return u

for L, p in (("lin", "/scratch/lm/lin_5g.arpa"), ("sna", "/scratch/lm/sna_5g.arpa")):
    U = arpa_uni(p)
    hw = [w for k, v in sub.items() if lang.get(k) == L for w in v.split()]
    oov = sum(1 for w in hw if w not in U)
    vw = [w for line in open("/scratch/prep/manifests/waxal_%s_validation.jsonl" % L, encoding="utf-8")
          if line.strip() for w in json.loads(line)["text"].split()]
    voov = sum(1 for w in vw if w not in U)
    print("%s | ARPA = %d unigrammes" % (L, len(U)))
    print("%s | tokens de NOTRE sortie hors-ARPA  : %d/%d = %.2f%%" % (L, oov, len(hw), 100.0*oov/len(hw)))
    print("%s | tokens de REFERENCE(val) hors-ARPA: %d/%d = %.2f%%" % (L, voov, len(vw), 100.0*voov/len(vw)))
    print("%s | deficit sur le test = %.0f tokens" % (L, (voov/len(vw) - oov/len(hw))*len(hw)))