Download recipe/textside/prep_text.py from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 2.83 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/textside/prep_text.py
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/recipe/textside/prep_text.py
-
curl -L -o prep_text.py https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/textside/prep_text.py
2.83 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright 2026 Kale | |
| """prep_text.py: the text-only distillation set for the Ettin text side (step 1a of textside/). | |
| Sentences from the text pools (config/text.json `pools`), split at sentence ends, deduplicated, 3-60 words, bench lines and every | |
| guard-line sentence held out, shuffled (seed 0); each sentence keeps the previous sentence of its passage (Ettin's left context). | |
| Token ids + word index per token come next, from make_ids.mjs (release front end, en-US and en-GB). | |
| Writes (under work/): | |
| sents.jsonl {text, prev} training sentences | |
| trace.jsonl {text, prev} the trace line's sentences (export trace + test inputs; config `trace_line`) | |
| check_texts.json [{id, text}] the first `check_lines` bench lines (export check inputs, check_items.mjs) | |
| usage: python3 -I prep_text.py [limit] [--config config/text.json] [--out work]""" | |
| import argparse | |
| import json | |
| import random | |
| import re | |
| from pathlib import Path | |
| HERE = Path(__file__).resolve().parent | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("limit", nargs="?", type=int, default=0) | |
| ap.add_argument("--config", default=str(HERE / "config" / "text.json")) | |
| ap.add_argument("--out", default=str(HERE / "work")) | |
| a = ap.parse_args() | |
| C = json.loads(Path(a.config).read_text()) | |
| rel = lambda p: (HERE / p).resolve() # noqa: E731 | |
| SENT = re.compile(C["sentence_split"]) | |
| def jsonl(p): | |
| return [json.loads(l) for l in open(p) if l.strip()] | |
| bench = [x for p in C["held_out_jsonl"] for x in jsonl(rel(p)) if "text" in x] | |
| guard = json.loads(rel(C["held_out_guard"]).read_text())["guard_lines"] # [[name, text], ...] | |
| held = {x["text"].strip() for x in bench} | |
| held |= {s.strip() for _, t in guard for s in SENT.split(t)} | |
| texts = [] | |
| for p in C["pools"]: | |
| texts += [x["text"] for x in jsonl(rel(p))] | |
| seen, items = set(), [] | |
| for t in texts: | |
| sents = [s.strip() for s in SENT.split(t.strip()) if s.strip()] | |
| prev = "" | |
| for s in sents: | |
| if s not in seen and s not in held and C["min_words"] <= len(s.split()) <= C["max_words"]: | |
| seen.add(s); items.append((s, prev)) | |
| prev = s | |
| random.Random(C["shuffle_seed"]).shuffle(items) | |
| lim = a.limit or len(items) | |
| od = Path(a.out); od.mkdir(parents=True, exist_ok=True) | |
| with open(od / "sents.jsonl", "w") as f: | |
| for s, prev in items[:lim]: | |
| f.write(json.dumps(dict(text=s, prev=prev)) + "\n") | |
| tl = dict(guard)[C["trace_line"]] | |
| with open(od / "trace.jsonl", "w") as f: | |
| prev = "" | |
| for s in [s.strip() for s in SENT.split(tl) if s.strip()]: | |
| f.write(json.dumps(dict(text=s, prev=prev)) + "\n"); prev = s | |
| (od / "check_texts.json").write_text(json.dumps([dict(id=x["id"], text=x["text"]) for x in bench[:C["check_lines"]]])) | |
| print("sentences", min(lim, len(items)), "of", len(items), "candidates; held out", len(held)) | |