File size: 945 Bytes
ae9e4fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env python3
import sys
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from mla.tokenizer import Tokenizer
TOK = Path("data/tokenizer/tokenizer.json")
SPLITS = {
"train": Path("data/splits/train.txt"),
"val": Path("data/splits/val.txt"),
}
OUT_DIR = Path("data/tokenized")
def main():
tok = Tokenizer.load(TOK)
OUT_DIR.mkdir(parents=True, exist_ok=True)
for name, path in SPLITS.items():
stream = []
n_dialogues = 0
for line in path.read_text(encoding="utf-8").splitlines():
if not line:
continue
stream.extend(tok.encode(line))
n_dialogues += 1
arr = np.array(stream, dtype=np.uint16)
out = OUT_DIR / f"{name}.npy"
np.save(out, arr)
print(f"{name}: dialogues={n_dialogues} tokens={len(arr)} -> {out}")
if __name__ == "__main__":
main()
|