| |
| import csv, json |
| sub = {} |
| for r in csv.DictReader(open("/root/sub_LMA06.csv", encoding="utf-8")): sub[r["ID"]] = r["Target"] |
| lang = json.load(open("/root/test_lang.json", encoding="utf-8")) |
|
|
| def arpa_uni(p): |
| u = set(); s = 0 |
| for line in open(p, encoding="utf-8", errors="replace"): |
| if line.startswith("\\1-grams:"): s = 1; continue |
| if line.startswith("\\2-grams:"): break |
| if s and line.strip(): |
| q = line.split("\t") |
| if len(q) >= 2: u.add(q[1]) |
| return u |
|
|
| for L, p in (("lin", "/scratch/lm/lin_5g.arpa"), ("sna", "/scratch/lm/sna_5g.arpa")): |
| U = arpa_uni(p) |
| hw = [w for k, v in sub.items() if lang.get(k) == L for w in v.split()] |
| oov = sum(1 for w in hw if w not in U) |
| vw = [w for line in open("/scratch/prep/manifests/waxal_%s_validation.jsonl" % L, encoding="utf-8") |
| if line.strip() for w in json.loads(line)["text"].split()] |
| voov = sum(1 for w in vw if w not in U) |
| print("%s | ARPA = %d unigrammes" % (L, len(U))) |
| print("%s | tokens de NOTRE sortie hors-ARPA : %d/%d = %.2f%%" % (L, oov, len(hw), 100.0*oov/len(hw))) |
| print("%s | tokens de REFERENCE(val) hors-ARPA: %d/%d = %.2f%%" % (L, voov, len(vw), 100.0*voov/len(vw))) |
| print("%s | deficit sur le test = %.0f tokens" % (L, (voov/len(vw) - oov/len(hw))*len(hw))) |
|
|
