| """Assemble a truecase release dir: weights + base tokenizer_config + truecase.py; run the helper on a few sentences and write a |
| lowercased, mark-free TED sample for the ONNX agreement check. python truecase/build_release_dir.py SRC BASE_ID OUT EVAL_DIR""" |
| import os, re, shutil, sys |
| import pandas as pd |
| from huggingface_hub import hf_hub_download |
| src, base, out, ev = sys.argv[1:5] |
| os.makedirs(out, exist_ok=True) |
| for f in ["config.json", "model.safetensors", "tokenizer.json", "train_log.json"]: |
| shutil.copy(f"{src}/{f}", f"{out}/{f}") |
| for f in ["tokenizer_config.json", "special_tokens_map.json"]: |
| shutil.copy(hf_hub_download(base, f), f"{out}/{f}") |
| shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "release", "truecase", "truecase.py"), f"{out}/truecase.py") |
| sys.path.insert(0, out) |
| from truecase import Truecaser, strip_marks, lower_keep_len |
| t = Truecaser(out) |
| for s in ["my name is anna i live in berlin and work for the united nations in new york", |
| "hello how are you today i am fine thanks and you", "the nasa mission launched on monday from florida", |
| "ich heiße thomas und wohne in münchen was machst du am wochenende", "hola me llamo maría y vivo en madrid con mi familia", |
| "je pense que paris est la plus belle ville de france", "привет меня зовут иван я живу в москве", |
| "今天天气很好我们去公园散步吧你觉得怎么样", "la reunión con el ministro garcía será el martes en bruselas"]: |
| print("helper |", t(s), flush=True) |
| d = pd.read_parquet(f"{ev}/ted.parquet") |
| d["text"] = [lower_keep_len(re.sub(r"\s+", " ", strip_marks(x))) for x in d.text] |
| d.to_parquet(os.path.join(os.path.dirname(out.rstrip("/")), "ted_lower.parquet"), index=False) |
|
|
