"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs, kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself. pip install onnxruntime numpy quickjs (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name) from kale_tts import KaleTTS tts = KaleTTS("models/kale-tts-1-nano") audio = tts.speak("A volcano is a place where very hot, melted rock comes up out of the ground.", voice="kenzie") # float32, 24 kHz for chunk in tts.stream(long_text, voice="d03"): play(chunk) # sentence by sentence """ import json, os, shutil, subprocess, threading from pathlib import Path import numpy as np import onnxruntime as ort SAMPLE_RATE = 24000 EN_GB_ON = True # (16 Nov: the 10-voice files are trained on en-gb ids for d03 / d13) British front end for en-gb voices (d03, d13); on only with weights trained on en-gb ids (= js/kale-tts.mjs EN_GB_ON) HERE = Path(__file__).resolve().parent def _near(name): """the release's frontend/ or models/ folder: next to this file (a host's own copy), or at the repo root (python/ sits one level below it, runtimes/python/ two), so both copies of this package are the same file""" for d in (HERE, *list(HERE.parents)[:2]): if (d / name).is_dir(): return d / name return HERE.parent / name class FrontEnd: """text -> sentences -> ids, via one long-lived `node frontend_cli.mjs` (thread-safe).""" def __init__(self, cli=HERE / "frontend_cli.mjs", node=None): node = node or os.environ.get("KALE_NODE") or shutil.which("node") if not node: raise RuntimeError("the text front end needs node (>= 18); or call speak_ids()/speak_ipa()") self.p = subprocess.Popen([node, str(cli)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1) self.lock = threading.Lock() def __call__(self, text, accent="en-us"): with self.lock: self.p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); self.p.stdin.flush(); r = json.loads(self.p.stdout.readline()) if "error" in r: raise ValueError(r["error"]) return r["sentences"] def close(self): self.p.terminate() class QuickJSFrontEnd: """the same front end without node: the bundled script (kale_frontend.js, built from the release front end) in QuickJS (pip install quickjs). Same output as FrontEnd. Thread-safe (one lock).""" def __init__(self, frontend_dir=None, bundle=HERE / "kale_frontend.js"): import base64, quickjs fd = Path(frontend_dir) if frontend_dir else _near("frontend") m = fd / "words"; self.ctx = quickjs.Context(); self.ctx.set_memory_limit(1 << 30); self.ctx.eval(Path(bundle).read_text()) self.ctx.eval("globalThis.kaleInit = (...a) => kaleFrontEnd.init(...a); globalThis.kaleInitGb = (...a) => kaleFrontEnd.initGb(...a);" "globalThis.kaleRun = (t, a) => kaleFrontEnd.run(t, a);") files = lambda d: [*[(d / f).read_text() for f in ("lexicon.tsv", "override.public.tsv", "terms.tsv")], # noqa: E731 base64.b64encode((d / "g2p.bin").read_bytes()).decode()] self.ctx.get("kaleInit")(*files(m)) self.accents = {"en-us"} if EN_GB_ON and (m / "en-gb/g2p.bin").exists(): # the British front end, as the JS API's EN_GB_ON self.ctx.get("kaleInitGb")(*files(m / "en-gb")); self.accents.add("en-gb") self.run = self.ctx.get("kaleRun"); self.lock = threading.Lock() self.ctx.eval("globalThis.kaleIpaWords = (t, a) => kaleFrontEnd.ipaWords(t, a); globalThis.kaleInitEttin = (j) => kaleFrontEnd.initEttin(j);" "globalThis.kaleEttin = (t, p, s, w) => kaleFrontEnd.ettin(t, p, s, w);") self.ettin_ready = False def init_ettin(self, tokenizer_json): with self.lock: self.ctx.get("kaleInitEttin")(tokenizer_json); self.ettin_ready = True def ipa_words(self, text, accent="en-us"): with self.lock: return json.loads(self.ctx.get("kaleIpaWords")(text, accent if accent in self.accents else "en-us")) def ettin(self, text, prev, sent_ids, word_ids): with self.lock: return json.loads(self.ctx.get("kaleEttin")(text, prev, json.dumps(sent_ids), json.dumps(word_ids))) def __call__(self, text, accent="en-us"): """-> [{text, ipa}] one per sentence; accents without a front end use en-us""" with self.lock: return json.loads(self.run(text, accent if accent in self.accents else "en-us")) def close(self): pass # Kokoro-native phoneme ids (voices.json "phonemes": "kokoro-native", the 17 Nov kale-tts-1-tiny) = js/kale-tts.mjs knIPA, line for line. _KN_PAIRS = {"eɪ": "A", "aɪ": "I", "aʊ": "W", "ɔɪ": "Y", "tʃ": "ʧ", "dʒ": "ʤ"} _KN_VOW = set("aeiouɑɐɒæəɘɚɛɜɝɞɪɨʊʉʌɔøœɤɯʏyAIWOYᵊᵻ"); _KN_END = set(" .,!?;:") def kn_ipa(chars, gb): """chars: the IPA characters that have ids -> the Kokoro-native characters (en-us rules, or the British convention when gb)""" pairs = {**_KN_PAIRS, **({"əʊ": "Q"} if gb else {"oʊ": "O"})}; out = []; i = 0 while i < len(chars): two = chars[i] + (chars[i + 1] if i + 1 < len(chars) else "") if two in pairs: out.append(pairs[two]); i += 2; continue c = chars[i]; i += 1 if gb: out.append(c); continue if c == "ː": continue out += ["ə", "ɹ"] if c == "ɚ" else ["ɜ", "ɹ"] if c == "ɝ" else ["T"] if c == "ɾ" else [c] for k in range(1, len(out) - 1): if out[k] == "ə" and out[k + 1] in "lnm" and (k + 2 == len(out) or out[k + 2] in _KN_END) and out[k - 1] not in _KN_VOW \ and out[k - 1] not in "ˈˌ" and not out[k - 1].isspace(): out[k] = "ᵊ" return out def default_frontend(): """QuickJS when installed (no node needed), else the node helper""" try: import quickjs # noqa: F401 return QuickJSFrontEnd() except ImportError: return FrontEnd() class KaleTTS: def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None): d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir) vj = json.loads((d / "voices.json").read_text()) model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph) so = ort.SessionOptions() if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1 self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers)) self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"] self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {} self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa) self.frontend = default_frontend() if frontend is True else (frontend or None) self.ettin = "ettin_ids" in [i.name for i in self.session.get_inputs()] # kale-tts-1-tiny's Ettin text side if self.ettin: if not isinstance(self.frontend, QuickJSFrontEnd): raise RuntimeError("this model's Ettin text side needs the QuickJS front end (pip install quickjs)") self.frontend.init_ettin((d / "ettin-tokenizer.json").read_text()) self.sample_rate = SAMPLE_RATE def list_voices(self): return [{"id": v["id"], "name": v.get("name", v["id"]), "description": v.get("description"), "accent": v.get("accent", "en-us"), "label": f"{v['name']} · {v['description']}" if v.get("description") else v.get("name", v["id"])} for v in self.voices] def resolve_voice(self, voice): """a voice by its public name (any case, e.g. "kenzie") or its internal id ("s057b") -> the id""" k = str(voice).strip().lower() for v in self.voices: if v["id"].lower() == k or v.get("name", "").lower() == k or k in v.get("aliases", []): return v["id"] # aliases: earlier names raise ValueError(f"unknown voice {voice}; voices: {', '.join(v.get('name', v['id']).lower() + ' (' + v['id'] + ')' for v in self.voices)}") def voice(self, vid=None): vid = self.resolve_voice(vid if vid is not None else self.voices[0]["id"]) if vid not in self._v: meta = next(v for v in self.voices if v["id"] == vid) sb, eb = (self.dir / f"voices/{vid}.bin").stat().st_size, (self.dir / f"voices/{vid}.extra.bin").stat().st_size need = sum(int(np.prod(vi["dims"])) for vi in self.voice_inputs) * 4 if sb == 0 or sb % (256 * 4): raise ValueError(f"voice {vid}: voices/{vid}.bin is {sb} bytes, not a style table of 256-float rows (wrong file?)") if eb != need: raise ValueError(f"voice {vid}: voices/{vid}.extra.bin is {eb} bytes, voices.json's voice_inputs need {need} (wrong file?)") st = np.fromfile(self.dir / f"voices/{vid}.bin", np.float32).reshape(-1, 256) ex, o, feeds = np.fromfile(self.dir / f"voices/{vid}.extra.bin", np.float32), 0, {} for vi in self.voice_inputs: n = int(np.prod(vi["dims"])); feeds[vi["name"]] = ex[o:o + n].reshape(vi["dims"]); o += n self._v[vid] = (meta, st, feeds) return self._v[vid] def speak_ids(self, ids, voice=None, ettin=None): """one sentence of Kokoro-table ids (no pads) -> float32 audio. ettin: this sentence's Ettin inputs (QuickJSFrontEnd.ettin), for models with the Ettin text side""" _, st, feeds = self.voice(voice); ids = list(ids)[:510] x = np.array([[0, *ids, 0]], np.int64) f = {"input_ids": x, "style": st[min(len(ids), len(st)) - 1][None], "speed": np.ones(1, np.float32), **feeds} if self.ettin: if ettin is None: raise ValueError("this model needs the sentence's Ettin inputs: use speak() / stream()") w = ettin["widx"][:len(ids)] f.update(ettin_ids=np.array([ettin["ids"]], np.int64), ettin_mask=np.ones((1, ettin["S"]), np.int64), wavg=np.array(ettin["wavg"], np.float32).reshape(1, ettin["W"], ettin["S"]), widx=np.array([[w[0], *w, w[-1]]], np.int64)) return self.session.run(None, f)[0][0] def _ids(self, ipa, accent="en-us"): chars = [c for c in ipa if c in self.vocab] return [self.vocab[c] for c in (kn_ipa(chars, accent == "en-gb") if self.kn else chars) if c in self.vocab] def speak_ipa(self, ipa, voice=None): return self.speak_ids(self._ids(ipa, self.voice(voice)[0].get("accent", "en-us")), voice) def stream(self, text, voice=None): """sentence by sentence: yields float32 audio as each sentence is ready""" if not self.frontend: raise RuntimeError("no front end: use speak_ids()/speak_ipa()") meta = self.voice(voice)[0]; acc = meta.get("accent", "en-us"); prev = "" for s in self.frontend(text, acc): ids = self._ids(s["ipa"], acc) if not ids: continue e = None if self.ettin: per = [self._ids(w, acc) for w in self.frontend.ipa_words(s["text"], acc)] e = self.frontend.ettin(s["text"], prev, ids, per) yield self.speak_ids(ids, voice, e); prev = s["text"] def speak(self, text, voice=None, gap_s=0.12): gap = np.zeros(int(gap_s * SAMPLE_RATE), np.float32); parts = [] for a in self.stream(text, voice): parts += [a, gap] return np.concatenate(parts) if parts else np.zeros(0, np.float32) def to_wav(audio, sr=SAMPLE_RATE): """16-bit PCM WAV bytes""" import io, wave b = io.BytesIO(); pcm = (np.clip(audio, -1, 1) * 32767).astype("