File size: 4,251 Bytes
9456b7b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Evaluation sets for punctuation restoration (evaluation only, never training). python punct/build_evals.py OUT_DIR (CPU job)
- flores: FLORES-200 devtest (CC-BY-SA-4.0): consecutive sentences of the same source article joined into one paragraph
(written news / travel / wiki text, all languages but those in common.SKIP).
- ted: TED2020 (OPUS; CC-BY-NC-ND-4.0) talk transcripts and their translations: 5 consecutive subtitle sentences, 200
paragraphs per language (spoken style).
- europarl: Europarl v8 (OPUS): 5 consecutive sentences, 200 per language, the 12 languages of kredor/punctuate-all.
oliverguhr/fullstop-* and kredor/punctuate-all were trained on Europarl, so this set is in-domain for them.
Output OUT_DIR/{set}.parquet: text (original, punctuated), lang (FLORES-style code), doc.
"""
import io, os, random, sys, tarfile, urllib.request, zipfile
import pandas as pd
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import SKIP, units
OUT = sys.argv[1]; os.makedirs(OUT, exist_ok=True)
rng = random.Random(0)
def get(url):
for k in range(5):
try:
return urllib.request.urlopen(url, timeout=900).read()
except Exception as e:
print("retry", url, e, flush=True)
raise RuntimeError(url)
# FLORES-200 devtest
tf = tarfile.open(fileobj=io.BytesIO(get("https://dl.fbaipublicfiles.com/nllb/flores200_dataset.tar.gz")))
files = {m.name: m for m in tf.getmembers() if m.isfile()}
meta = next(n for n in files if n.endswith("metadata_devtest.tsv"))
md = pd.read_csv(tf.extractfile(files[meta]), sep="\t")
urls = md.URL.tolist()
rows = []
for n in sorted(files):
if "/devtest/" not in n or not n.endswith(".devtest"):
continue
code = os.path.basename(n)[:-8]
if code in SKIP:
continue
sents = tf.extractfile(files[n]).read().decode("utf-8").split("\n")[:len(urls)]
cur, doc = [], None
for s, u in zip(sents, urls):
if u != doc and cur:
rows.append((("" if code.startswith(("zho", "jpn", "yue")) else " ").join(cur), code, doc)); cur = []
cur.append(s.strip()); doc = u
if cur:
rows.append((" ".join(cur), code, doc))
pd.DataFrame(rows, columns=["text", "lang", "doc"]).to_parquet(f"{OUT}/flores.parquet")
print("flores", len(rows), len({r[1] for r in rows}), flush=True)
def opus(corpus, ver, l2, code, n=200, k=5):
a, b = sorted([l2, "en"]) if l2 != "en" else ("de", "en")
z = zipfile.ZipFile(io.BytesIO(get(f"https://object.pouta.csc.fi/OPUS-{corpus}/{ver}/moses/{a}-{b}.txt.zip")))
name = next(x for x in z.namelist() if x.endswith(f".{a}-{b}.{l2}"))
lines = z.read(name).decode("utf-8").split("\n")
out, tries = [], 0
while len(out) < n and tries < 50 * n:
tries += 1; i = rng.randrange(0, len(lines) - k)
par = [x.strip() for x in lines[i:i + k]]
if any(len(x) < 3 for x in par) or any(units(x, code)[2][-1:] in ([], ["0"]) for x in par):
continue # empty lines, headings, lines without final punctuation
out.append((("" if code.startswith(("zho", "jpn")) else " ").join(par), code, f"{corpus}:{i}"))
print(corpus, code, len(out), flush=True)
return out
TED = dict(en="eng_Latn", de="deu_Latn", fr="fra_Latn", es="spa_Latn", it="ita_Latn", pt="por_Latn", nl="nld_Latn", pl="pol_Latn",
ru="rus_Cyrl", uk="ukr_Cyrl", cs="ces_Latn", ro="ron_Latn", hu="hun_Latn", el="ell_Grek", bg="bul_Cyrl", tr="tur_Latn",
ar="arb_Arab", he="heb_Hebr", fa="pes_Arab", hi="hin_Deva", id="ind_Latn", vi="vie_Latn", ja="jpn_Jpan", ko="kor_Hang",
zh_cn="zho_Hans")
EP = dict(en="eng_Latn", de="deu_Latn", fr="fra_Latn", es="spa_Latn", it="ita_Latn", pt="por_Latn", nl="nld_Latn", pl="pol_Latn",
cs="ces_Latn", bg="bul_Cyrl", sk="slk_Latn", sl="slv_Latn")
for corpus, ver, langs, name in [("TED2020", "v1", TED, "ted"), ("Europarl", "v8", EP, "europarl")]:
rows = []
for l2, code in langs.items():
try:
rows += opus(corpus, ver, l2, code)
except Exception as e:
print("FAILED", corpus, l2, e, flush=True)
pd.DataFrame(rows, columns=["text", "lang", "doc"]).to_parquet(f"{OUT}/{name}.parquet")
print("done", flush=True)
|