kale-tts-1-tiny / recipe /textside /prep_text.py
NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
2.83 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""prep_text.py: the text-only distillation set for the Ettin text side (step 1a of textside/).
Sentences from the text pools (config/text.json `pools`), split at sentence ends, deduplicated, 3-60 words, bench lines and every
guard-line sentence held out, shuffled (seed 0); each sentence keeps the previous sentence of its passage (Ettin's left context).
Token ids + word index per token come next, from make_ids.mjs (release front end, en-US and en-GB).
Writes (under work/):
sents.jsonl {text, prev} training sentences
trace.jsonl {text, prev} the trace line's sentences (export trace + test inputs; config `trace_line`)
check_texts.json [{id, text}] the first `check_lines` bench lines (export check inputs, check_items.mjs)
usage: python3 -I prep_text.py [limit] [--config config/text.json] [--out work]"""
import argparse
import json
import random
import re
from pathlib import Path
HERE = Path(__file__).resolve().parent
ap = argparse.ArgumentParser()
ap.add_argument("limit", nargs="?", type=int, default=0)
ap.add_argument("--config", default=str(HERE / "config" / "text.json"))
ap.add_argument("--out", default=str(HERE / "work"))
a = ap.parse_args()
C = json.loads(Path(a.config).read_text())
rel = lambda p: (HERE / p).resolve() # noqa: E731
SENT = re.compile(C["sentence_split"])
def jsonl(p):
return [json.loads(l) for l in open(p) if l.strip()]
bench = [x for p in C["held_out_jsonl"] for x in jsonl(rel(p)) if "text" in x]
guard = json.loads(rel(C["held_out_guard"]).read_text())["guard_lines"] # [[name, text], ...]
held = {x["text"].strip() for x in bench}
held |= {s.strip() for _, t in guard for s in SENT.split(t)}
texts = []
for p in C["pools"]:
texts += [x["text"] for x in jsonl(rel(p))]
seen, items = set(), []
for t in texts:
sents = [s.strip() for s in SENT.split(t.strip()) if s.strip()]
prev = ""
for s in sents:
if s not in seen and s not in held and C["min_words"] <= len(s.split()) <= C["max_words"]:
seen.add(s); items.append((s, prev))
prev = s
random.Random(C["shuffle_seed"]).shuffle(items)
lim = a.limit or len(items)
od = Path(a.out); od.mkdir(parents=True, exist_ok=True)
with open(od / "sents.jsonl", "w") as f:
for s, prev in items[:lim]:
f.write(json.dumps(dict(text=s, prev=prev)) + "\n")
tl = dict(guard)[C["trace_line"]]
with open(od / "trace.jsonl", "w") as f:
prev = ""
for s in [s.strip() for s in SENT.split(tl) if s.strip()]:
f.write(json.dumps(dict(text=s, prev=prev)) + "\n"); prev = s
(od / "check_texts.json").write_text(json.dumps([dict(id=x["id"], text=x["text"]) for x in bench[:C["check_lines"]]]))
print("sentences", min(lim, len(items)), "of", len(items), "candidates; held out", len(held))