File size: 1,443 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | #!/usr/bin/env python3
"""Applique le fill-empty (leophill) a un CSV: les clips a Target <=2 mots deviennent
le 4-gram le plus frequent du train de leur langue. Gain valide +0.0025 sur la val."""
import csv
import json
import sys
from collections import Counter
def norm(t):
return " ".join(str(t).replace("|", " ").split())
def top_gram(lang):
rows = [json.loads(l) for l in open(f"/scratch/prep/manifests/waxal_{lang}_train.jsonl", encoding="utf-8")]
texts = [norm(r["text"]).lower() for r in rows if r["text"].strip()]
grams = Counter()
for t in texts:
w = t.split()
for i in range(len(w) - 3):
grams[" ".join(w[i:i+4])] += 1
return grams.most_common(1)[0][0] if grams else "a"
def main():
inp, out = sys.argv[1], sys.argv[2]
priors = {l: top_gram(l) for l in ("lin", "lug", "sna")}
print("priors:", priors)
rows = list(csv.DictReader(open(inp, encoding="utf-8")))
n = 0
for r in rows:
lang = r["ID"].split("_")[0]
if len(norm(r["Target"]).split()) <= 2 and lang in priors:
r["Target"] = priors[lang]
n += 1
with open(out, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["ID", "Target"])
for r in rows:
w.writerow([r["ID"], r["Target"] or "a"])
print(f"FILL_APPLIED {out}: {n} clips remplaces sur {len(rows)}")
if __name__ == "__main__":
main()
|