# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Kale """make_ids.py: text -> Kokoro-table token ids for a voice's training takes, through the RELEASE text front end (../../python/frontend_cli.mjs: sentence split + normaliser + CMUdict lexicon / overrides + g2p -> ids; no espeak). A take's ids = its sentences' ids joined by the word-space id (16), no 0 pads: the layout our training id files had (checked against the release build's own id files on passages). The accent comes from config/voices.json `front_end` (d03 / d13 en-GB, others en-US). Writes, under /work/ (root = env KALE_TTS_ROOT or "."): ids__all.json {"rows": {clip: [ids]}} (train/prep.py input) ids__text.json {text: [ids]} (with --text-map; teacher/make_rows.py input) Same role as the id step of our build (our earlier Python copy of the same lexicon + g2p); a few takes may differ by an id or two. usage: python3 make_ids.py [--jobs work/jobs-.jsonl] [--accent en-us|en-gb] [--text-map] [--frontend ../../python/frontend_cli.mjs] [--node node] (needs node >= 18, no npm packages)""" import argparse import json import os import shutil import subprocess from pathlib import Path HERE = Path(__file__).resolve().parent SPACE_ID = 16 ap = argparse.ArgumentParser() ap.add_argument("voice") ap.add_argument("--jobs", default="") ap.add_argument("--accent", default="", help="default: config/voices.json front_end of the voice") ap.add_argument("--text-map", action="store_true") ap.add_argument("--frontend", default=str(HERE.parent.parent / "python" / "frontend_cli.mjs")) ap.add_argument("--node", default=shutil.which("node") or "node") a = ap.parse_args() root = Path(os.environ.get("KALE_TTS_ROOT", ".")) cfg = json.loads((HERE / "config" / "voices.json").read_text()) vc = {v["id"]: v for v in cfg["voices"] + cfg.get("teacher_only", [])}.get(a.voice, {}) accent = a.accent or vc.get("front_end", "en-US").lower() jobs = [json.loads(l) for l in Path(a.jobs or root / f"work/jobs-{a.voice}.jsonl").read_text().splitlines() if l.strip()] p = subprocess.Popen([a.node, a.frontend], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1) def ids(text): p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); p.stdin.flush() r = json.loads(p.stdout.readline()) if "error" in r: raise ValueError(f"{text[:60]}: {r['error']}") out = [] for k, s in enumerate(r["sentences"]): out += ([SPACE_ID] if k else []) + s["ids"] return out rows, by_text, bad = {}, {}, 0 for j in jobs: t = j["text"] if t not in by_text: try: by_text[t] = ids(t) except ValueError as e: bad += 1; print("skip", e); continue rows[j["clip"]] = by_text[t] p.terminate() (root / "work").mkdir(parents=True, exist_ok=True) (root / f"work/ids_{a.voice}_all.json").write_text(json.dumps(dict(rows=rows))) if a.text_map: (root / f"work/ids_{a.voice}_text.json").write_text(json.dumps(by_text)) print(a.voice, accent, len(rows), "takes,", bad, "skipped")