File size: 3,090 Bytes
f21d6eb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""make_ids.py: text -> Kokoro-table token ids for a voice's training takes, through the RELEASE text front end
(../../python/frontend_cli.mjs: sentence split + normaliser + CMUdict lexicon / overrides + g2p -> ids; no espeak).
A take's ids = its sentences' ids joined by the word-space id (16), no 0 pads: the layout our training id files had (checked against
the release build's own id files on passages). The accent comes from config/voices.json `front_end` (d03 / d13 en-GB, others en-US).
Writes, under <root>/work/ (root = env KALE_TTS_ROOT or "."):
  ids_<voice>_all.json    {"rows": {clip: [ids]}}   (train/prep.py input)
  ids_<voice>_text.json   {text: [ids]}             (with --text-map; teacher/make_rows.py input)
Same role as the id step of our build (our earlier Python copy of the same lexicon + g2p); a few takes may differ by an id or two.
usage: python3 make_ids.py <voice> [--jobs work/jobs-<voice>.jsonl] [--accent en-us|en-gb] [--text-map]
       [--frontend ../../python/frontend_cli.mjs] [--node node]      (needs node >= 18, no npm packages)"""
import argparse
import json
import os
import shutil
import subprocess
from pathlib import Path

HERE = Path(__file__).resolve().parent
SPACE_ID = 16
ap = argparse.ArgumentParser()
ap.add_argument("voice")
ap.add_argument("--jobs", default="")
ap.add_argument("--accent", default="", help="default: config/voices.json front_end of the voice")
ap.add_argument("--text-map", action="store_true")
ap.add_argument("--frontend", default=str(HERE.parent.parent / "python" / "frontend_cli.mjs"))
ap.add_argument("--node", default=shutil.which("node") or "node")
a = ap.parse_args()
root = Path(os.environ.get("KALE_TTS_ROOT", "."))
cfg = json.loads((HERE / "config" / "voices.json").read_text())
vc = {v["id"]: v for v in cfg["voices"] + cfg.get("teacher_only", [])}.get(a.voice, {})
accent = a.accent or vc.get("front_end", "en-US").lower()
jobs = [json.loads(l) for l in Path(a.jobs or root / f"work/jobs-{a.voice}.jsonl").read_text().splitlines() if l.strip()]
p = subprocess.Popen([a.node, a.frontend], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1)


def ids(text):
    p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); p.stdin.flush()
    r = json.loads(p.stdout.readline())
    if "error" in r:
        raise ValueError(f"{text[:60]}: {r['error']}")
    out = []
    for k, s in enumerate(r["sentences"]):
        out += ([SPACE_ID] if k else []) + s["ids"]
    return out


rows, by_text, bad = {}, {}, 0
for j in jobs:
    t = j["text"]
    if t not in by_text:
        try:
            by_text[t] = ids(t)
        except ValueError as e:
            bad += 1; print("skip", e); continue
    rows[j["clip"]] = by_text[t]
p.terminate()
(root / "work").mkdir(parents=True, exist_ok=True)
(root / f"work/ids_{a.voice}_all.json").write_text(json.dumps(dict(rows=rows)))
if a.text_map:
    (root / f"work/ids_{a.voice}_text.json").write_text(json.dumps(by_text))
print(a.voice, accent, len(rows), "takes,", bad, "skipped")