File size: 1,781 Bytes
45abde7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | """Assemble a truecase release dir: weights + base tokenizer_config + truecase.py; run the helper on a few sentences and write a
lowercased, mark-free TED sample for the ONNX agreement check. python truecase/build_release_dir.py SRC BASE_ID OUT EVAL_DIR"""
import os, re, shutil, sys
import pandas as pd
from huggingface_hub import hf_hub_download
src, base, out, ev = sys.argv[1:5]
os.makedirs(out, exist_ok=True)
for f in ["config.json", "model.safetensors", "tokenizer.json", "train_log.json"]:
shutil.copy(f"{src}/{f}", f"{out}/{f}")
for f in ["tokenizer_config.json", "special_tokens_map.json"]:
shutil.copy(hf_hub_download(base, f), f"{out}/{f}")
shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "release", "truecase", "truecase.py"), f"{out}/truecase.py")
sys.path.insert(0, out)
from truecase import Truecaser, strip_marks, lower_keep_len
t = Truecaser(out)
for s in ["my name is anna i live in berlin and work for the united nations in new york",
"hello how are you today i am fine thanks and you", "the nasa mission launched on monday from florida",
"ich heiße thomas und wohne in münchen was machst du am wochenende", "hola me llamo maría y vivo en madrid con mi familia",
"je pense que paris est la plus belle ville de france", "привет меня зовут иван я живу в москве",
"今天天气很好我们去公园散步吧你觉得怎么样", "la reunión con el ministro garcía será el martes en bruselas"]:
print("helper |", t(s), flush=True)
d = pd.read_parquet(f"{ev}/ted.parquet")
d["text"] = [lower_keep_len(re.sub(r"\s+", " ", strip_marks(x))) for x in d.text]
d.to_parquet(os.path.join(os.path.dirname(out.rstrip("/")), "ted_lower.parquet"), index=False)
|