File size: 12,157 Bytes
f21d6eb
 
6901100
f21d6eb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6901100
 
f21d6eb
 
 
6901100
f21d6eb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
pip install onnxruntime numpy quickjs   (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name)

    from kale_tts import KaleTTS
    tts = KaleTTS("models/kale-tts-1-nano")
    audio = tts.speak("A volcano is a place where very hot, melted rock comes up out of the ground.", voice="kenzie")   # float32, 24 kHz
    for chunk in tts.stream(long_text, voice="d03"): play(chunk)                                                   # sentence by sentence
"""
import json, os, shutil, subprocess, threading
from pathlib import Path
import numpy as np
import onnxruntime as ort

SAMPLE_RATE = 24000
EN_GB_ON = True   # (16 Nov: the 10-voice files are trained on en-gb ids for d03 / d13) British front end for en-gb voices (d03, d13); on only with weights trained on en-gb ids (= js/kale-tts.mjs EN_GB_ON)
HERE = Path(__file__).resolve().parent


def _near(name):
    """the release's frontend/ or models/ folder: next to this file (a host's own copy), or at the repo root (python/ sits one level
    below it, runtimes/python/ two), so both copies of this package are the same file"""
    for d in (HERE, *list(HERE.parents)[:2]):
        if (d / name).is_dir(): return d / name
    return HERE.parent / name


class FrontEnd:
    """text -> sentences -> ids, via one long-lived `node frontend_cli.mjs` (thread-safe)."""
    def __init__(self, cli=HERE / "frontend_cli.mjs", node=None):
        node = node or os.environ.get("KALE_NODE") or shutil.which("node")
        if not node: raise RuntimeError("the text front end needs node (>= 18); or call speak_ids()/speak_ipa()")
        self.p = subprocess.Popen([node, str(cli)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1)
        self.lock = threading.Lock()
    def __call__(self, text, accent="en-us"):
        with self.lock:
            self.p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); self.p.stdin.flush(); r = json.loads(self.p.stdout.readline())
        if "error" in r: raise ValueError(r["error"])
        return r["sentences"]
    def close(self): self.p.terminate()


class QuickJSFrontEnd:
    """the same front end without node: the bundled script (kale_frontend.js, built from the release front end) in QuickJS
    (pip install quickjs). Same output as FrontEnd. Thread-safe (one lock)."""
    def __init__(self, frontend_dir=None, bundle=HERE / "kale_frontend.js"):
        import base64, quickjs
        fd = Path(frontend_dir) if frontend_dir else _near("frontend")
        m = fd / "words"; self.ctx = quickjs.Context(); self.ctx.set_memory_limit(1 << 30); self.ctx.eval(Path(bundle).read_text())
        self.ctx.eval("globalThis.kaleInit = (...a) => kaleFrontEnd.init(...a); globalThis.kaleInitGb = (...a) => kaleFrontEnd.initGb(...a);"
                      "globalThis.kaleRun = (t, a) => kaleFrontEnd.run(t, a);")
        files = lambda d: [*[(d / f).read_text() for f in ("lexicon.tsv", "override.public.tsv", "terms.tsv")],  # noqa: E731
                           base64.b64encode((d / "g2p.bin").read_bytes()).decode()]
        self.ctx.get("kaleInit")(*files(m))
        self.accents = {"en-us"}
        if EN_GB_ON and (m / "en-gb/g2p.bin").exists():   # the British front end, as the JS API's EN_GB_ON
            self.ctx.get("kaleInitGb")(*files(m / "en-gb")); self.accents.add("en-gb")
        self.run = self.ctx.get("kaleRun"); self.lock = threading.Lock()
        self.ctx.eval("globalThis.kaleIpaWords = (t, a) => kaleFrontEnd.ipaWords(t, a); globalThis.kaleInitEttin = (j) => kaleFrontEnd.initEttin(j);"
                      "globalThis.kaleEttin = (t, p, s, w) => kaleFrontEnd.ettin(t, p, s, w);")
        self.ettin_ready = False
    def init_ettin(self, tokenizer_json):
        with self.lock: self.ctx.get("kaleInitEttin")(tokenizer_json); self.ettin_ready = True
    def ipa_words(self, text, accent="en-us"):
        with self.lock: return json.loads(self.ctx.get("kaleIpaWords")(text, accent if accent in self.accents else "en-us"))
    def ettin(self, text, prev, sent_ids, word_ids):
        with self.lock: return json.loads(self.ctx.get("kaleEttin")(text, prev, json.dumps(sent_ids), json.dumps(word_ids)))
    def __call__(self, text, accent="en-us"):
        """-> [{text, ipa}] one per sentence; accents without a front end use en-us"""
        with self.lock: return json.loads(self.run(text, accent if accent in self.accents else "en-us"))
    def close(self): pass


# Kokoro-native phoneme ids (voices.json "phonemes": "kokoro-native", the 17 Nov kale-tts-1-tiny) = js/kale-tts.mjs knIPA, line for line.
_KN_PAIRS = {"eɪ": "A", "aɪ": "I", "aʊ": "W", "ɔɪ": "Y", "tʃ": "ʧ", "dʒ": "ʤ"}
_KN_VOW = set("aeiouɑɐɒæəɘɚɛɜɝɞɪɨʊʉʌɔøœɤɯʏyAIWOYᵊᵻ"); _KN_END = set(" .,!?;:")


def kn_ipa(chars, gb):
    """chars: the IPA characters that have ids -> the Kokoro-native characters (en-us rules, or the British convention when gb)"""
    pairs = {**_KN_PAIRS, **({"əʊ": "Q"} if gb else {"oʊ": "O"})}; out = []; i = 0
    while i < len(chars):
        two = chars[i] + (chars[i + 1] if i + 1 < len(chars) else "")
        if two in pairs: out.append(pairs[two]); i += 2; continue
        c = chars[i]; i += 1
        if gb: out.append(c); continue
        if c == "ː": continue
        out += ["ə", "ɹ"] if c == "ɚ" else ["ɜ", "ɹ"] if c == "ɝ" else ["T"] if c == "ɾ" else [c]
    for k in range(1, len(out) - 1):
        if out[k] == "ə" and out[k + 1] in "lnm" and (k + 2 == len(out) or out[k + 2] in _KN_END) and out[k - 1] not in _KN_VOW \
                and out[k - 1] not in "ˈˌ" and not out[k - 1].isspace():
            out[k] = "ᵊ"
    return out


def default_frontend():
    """QuickJS when installed (no node needed), else the node helper"""
    try:
        import quickjs  # noqa: F401
        return QuickJSFrontEnd()
    except ImportError:
        return FrontEnd()


class KaleTTS:
    def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
        d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
        vj = json.loads((d / "voices.json").read_text())
        model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph)
        so = ort.SessionOptions()
        if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
        self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
        self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"]
        self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
        self.kn = vj.get("phonemes") == "kokoro-native"   # Kokoro-native ids (see kn_ipa)
        self.frontend = default_frontend() if frontend is True else (frontend or None)
        self.ettin = "ettin_ids" in [i.name for i in self.session.get_inputs()]   # kale-tts-1-tiny's Ettin text side
        if self.ettin:
            if not isinstance(self.frontend, QuickJSFrontEnd): raise RuntimeError("this model's Ettin text side needs the QuickJS front end (pip install quickjs)")
            self.frontend.init_ettin((d / "ettin-tokenizer.json").read_text())
        self.sample_rate = SAMPLE_RATE

    def list_voices(self):
        return [{"id": v["id"], "name": v.get("name", v["id"]), "description": v.get("description"), "accent": v.get("accent", "en-us"),
                 "label": f"{v['name']} · {v['description']}" if v.get("description") else v.get("name", v["id"])} for v in self.voices]

    def resolve_voice(self, voice):
        """a voice by its public name (any case, e.g. "kenzie") or its internal id ("s057b") -> the id"""
        k = str(voice).strip().lower()
        for v in self.voices:
            if v["id"].lower() == k or v.get("name", "").lower() == k or k in v.get("aliases", []): return v["id"]   # aliases: earlier names
        raise ValueError(f"unknown voice {voice}; voices: {', '.join(v.get('name', v['id']).lower() + ' (' + v['id'] + ')' for v in self.voices)}")

    def voice(self, vid=None):
        vid = self.resolve_voice(vid if vid is not None else self.voices[0]["id"])
        if vid not in self._v:
            meta = next(v for v in self.voices if v["id"] == vid)
            sb, eb = (self.dir / f"voices/{vid}.bin").stat().st_size, (self.dir / f"voices/{vid}.extra.bin").stat().st_size
            need = sum(int(np.prod(vi["dims"])) for vi in self.voice_inputs) * 4
            if sb == 0 or sb % (256 * 4): raise ValueError(f"voice {vid}: voices/{vid}.bin is {sb} bytes, not a style table of 256-float rows (wrong file?)")
            if eb != need: raise ValueError(f"voice {vid}: voices/{vid}.extra.bin is {eb} bytes, voices.json's voice_inputs need {need} (wrong file?)")
            st = np.fromfile(self.dir / f"voices/{vid}.bin", np.float32).reshape(-1, 256)
            ex, o, feeds = np.fromfile(self.dir / f"voices/{vid}.extra.bin", np.float32), 0, {}
            for vi in self.voice_inputs:
                n = int(np.prod(vi["dims"])); feeds[vi["name"]] = ex[o:o + n].reshape(vi["dims"]); o += n
            self._v[vid] = (meta, st, feeds)
        return self._v[vid]

    def speak_ids(self, ids, voice=None, ettin=None):
        """one sentence of Kokoro-table ids (no pads) -> float32 audio. ettin: this sentence's Ettin inputs (QuickJSFrontEnd.ettin), for
        models with the Ettin text side"""
        _, st, feeds = self.voice(voice); ids = list(ids)[:510]
        x = np.array([[0, *ids, 0]], np.int64)
        f = {"input_ids": x, "style": st[min(len(ids), len(st)) - 1][None], "speed": np.ones(1, np.float32), **feeds}
        if self.ettin:
            if ettin is None: raise ValueError("this model needs the sentence's Ettin inputs: use speak() / stream()")
            w = ettin["widx"][:len(ids)]
            f.update(ettin_ids=np.array([ettin["ids"]], np.int64), ettin_mask=np.ones((1, ettin["S"]), np.int64),
                     wavg=np.array(ettin["wavg"], np.float32).reshape(1, ettin["W"], ettin["S"]), widx=np.array([[w[0], *w, w[-1]]], np.int64))
        return self.session.run(None, f)[0][0]

    def _ids(self, ipa, accent="en-us"):
        chars = [c for c in ipa if c in self.vocab]
        return [self.vocab[c] for c in (kn_ipa(chars, accent == "en-gb") if self.kn else chars) if c in self.vocab]

    def speak_ipa(self, ipa, voice=None): return self.speak_ids(self._ids(ipa, self.voice(voice)[0].get("accent", "en-us")), voice)

    def stream(self, text, voice=None):
        """sentence by sentence: yields float32 audio as each sentence is ready"""
        if not self.frontend: raise RuntimeError("no front end: use speak_ids()/speak_ipa()")
        meta = self.voice(voice)[0]; acc = meta.get("accent", "en-us"); prev = ""
        for s in self.frontend(text, acc):
            ids = self._ids(s["ipa"], acc)
            if not ids: continue
            e = None
            if self.ettin:
                per = [self._ids(w, acc) for w in self.frontend.ipa_words(s["text"], acc)]
                e = self.frontend.ettin(s["text"], prev, ids, per)
            yield self.speak_ids(ids, voice, e); prev = s["text"]

    def speak(self, text, voice=None, gap_s=0.12):
        gap = np.zeros(int(gap_s * SAMPLE_RATE), np.float32); parts = []
        for a in self.stream(text, voice): parts += [a, gap]
        return np.concatenate(parts) if parts else np.zeros(0, np.float32)


def to_wav(audio, sr=SAMPLE_RATE):
    """16-bit PCM WAV bytes"""
    import io, wave
    b = io.BytesIO(); pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2")
    with wave.open(b, "wb") as w: w.setnchannels(1); w.setsampwidth(2); w.setframerate(sr); w.writeframes(pcm.tobytes())
    return b.getvalue()


def to_pcm16(audio): return (np.clip(audio, -1, 1) * 32767).astype("<i2").tobytes()