kale-tts-1-tiny / python /kale_tts.py
NERDDISCO's picture
Clear file names: kale-tts-1-tiny.onnx at the root, PyTorch weights in pytorch/
6901100 verified
Raw History Blame Contribute Delete
12.2 kB
"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
pip install onnxruntime numpy quickjs (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name)
from kale_tts import KaleTTS
tts = KaleTTS("models/kale-tts-1-nano")
audio = tts.speak("A volcano is a place where very hot, melted rock comes up out of the ground.", voice="kenzie") # float32, 24 kHz
for chunk in tts.stream(long_text, voice="d03"): play(chunk) # sentence by sentence
"""
import json, os, shutil, subprocess, threading
from pathlib import Path
import numpy as np
import onnxruntime as ort
SAMPLE_RATE = 24000
EN_GB_ON = True # (16 Nov: the 10-voice files are trained on en-gb ids for d03 / d13) British front end for en-gb voices (d03, d13); on only with weights trained on en-gb ids (= js/kale-tts.mjs EN_GB_ON)
HERE = Path(__file__).resolve().parent
def _near(name):
"""the release's frontend/ or models/ folder: next to this file (a host's own copy), or at the repo root (python/ sits one level
below it, runtimes/python/ two), so both copies of this package are the same file"""
for d in (HERE, *list(HERE.parents)[:2]):
if (d / name).is_dir(): return d / name
return HERE.parent / name
class FrontEnd:
"""text -> sentences -> ids, via one long-lived `node frontend_cli.mjs` (thread-safe)."""
def __init__(self, cli=HERE / "frontend_cli.mjs", node=None):
node = node or os.environ.get("KALE_NODE") or shutil.which("node")
if not node: raise RuntimeError("the text front end needs node (>= 18); or call speak_ids()/speak_ipa()")
self.p = subprocess.Popen([node, str(cli)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1)
self.lock = threading.Lock()
def __call__(self, text, accent="en-us"):
with self.lock:
self.p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); self.p.stdin.flush(); r = json.loads(self.p.stdout.readline())
if "error" in r: raise ValueError(r["error"])
return r["sentences"]
def close(self): self.p.terminate()
class QuickJSFrontEnd:
"""the same front end without node: the bundled script (kale_frontend.js, built from the release front end) in QuickJS
(pip install quickjs). Same output as FrontEnd. Thread-safe (one lock)."""
def __init__(self, frontend_dir=None, bundle=HERE / "kale_frontend.js"):
import base64, quickjs
fd = Path(frontend_dir) if frontend_dir else _near("frontend")
m = fd / "words"; self.ctx = quickjs.Context(); self.ctx.set_memory_limit(1 << 30); self.ctx.eval(Path(bundle).read_text())
self.ctx.eval("globalThis.kaleInit = (...a) => kaleFrontEnd.init(...a); globalThis.kaleInitGb = (...a) => kaleFrontEnd.initGb(...a);"
"globalThis.kaleRun = (t, a) => kaleFrontEnd.run(t, a);")
files = lambda d: [*[(d / f).read_text() for f in ("lexicon.tsv", "override.public.tsv", "terms.tsv")], # noqa: E731
base64.b64encode((d / "g2p.bin").read_bytes()).decode()]
self.ctx.get("kaleInit")(*files(m))
self.accents = {"en-us"}
if EN_GB_ON and (m / "en-gb/g2p.bin").exists(): # the British front end, as the JS API's EN_GB_ON
self.ctx.get("kaleInitGb")(*files(m / "en-gb")); self.accents.add("en-gb")
self.run = self.ctx.get("kaleRun"); self.lock = threading.Lock()
self.ctx.eval("globalThis.kaleIpaWords = (t, a) => kaleFrontEnd.ipaWords(t, a); globalThis.kaleInitEttin = (j) => kaleFrontEnd.initEttin(j);"
"globalThis.kaleEttin = (t, p, s, w) => kaleFrontEnd.ettin(t, p, s, w);")
self.ettin_ready = False
def init_ettin(self, tokenizer_json):
with self.lock: self.ctx.get("kaleInitEttin")(tokenizer_json); self.ettin_ready = True
def ipa_words(self, text, accent="en-us"):
with self.lock: return json.loads(self.ctx.get("kaleIpaWords")(text, accent if accent in self.accents else "en-us"))
def ettin(self, text, prev, sent_ids, word_ids):
with self.lock: return json.loads(self.ctx.get("kaleEttin")(text, prev, json.dumps(sent_ids), json.dumps(word_ids)))
def __call__(self, text, accent="en-us"):
"""-> [{text, ipa}] one per sentence; accents without a front end use en-us"""
with self.lock: return json.loads(self.run(text, accent if accent in self.accents else "en-us"))
def close(self): pass
# Kokoro-native phoneme ids (voices.json "phonemes": "kokoro-native", the 17 Nov kale-tts-1-tiny) = js/kale-tts.mjs knIPA, line for line.
_KN_PAIRS = {"eɪ": "A", "aɪ": "I", "aʊ": "W", "ɔɪ": "Y", "tʃ": "ʧ", "dʒ": "ʤ"}
_KN_VOW = set("aeiouɑɐɒæəɘɚɛɜɝɞɪɨʊʉʌɔøœɤɯʏyAIWOYᵊᵻ"); _KN_END = set(" .,!?;:")
def kn_ipa(chars, gb):
"""chars: the IPA characters that have ids -> the Kokoro-native characters (en-us rules, or the British convention when gb)"""
pairs = {**_KN_PAIRS, **({"əʊ": "Q"} if gb else {"oʊ": "O"})}; out = []; i = 0
while i < len(chars):
two = chars[i] + (chars[i + 1] if i + 1 < len(chars) else "")
if two in pairs: out.append(pairs[two]); i += 2; continue
c = chars[i]; i += 1
if gb: out.append(c); continue
if c == "ː": continue
out += ["ə", "ɹ"] if c == "ɚ" else ["ɜ", "ɹ"] if c == "ɝ" else ["T"] if c == "ɾ" else [c]
for k in range(1, len(out) - 1):
if out[k] == "ə" and out[k + 1] in "lnm" and (k + 2 == len(out) or out[k + 2] in _KN_END) and out[k - 1] not in _KN_VOW \
and out[k - 1] not in "ˈˌ" and not out[k - 1].isspace():
out[k] = "ᵊ"
return out
def default_frontend():
"""QuickJS when installed (no node needed), else the node helper"""
try:
import quickjs # noqa: F401
return QuickJSFrontEnd()
except ImportError:
return FrontEnd()
class KaleTTS:
def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
vj = json.loads((d / "voices.json").read_text())
model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph)
so = ort.SessionOptions()
if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"]
self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa)
self.frontend = default_frontend() if frontend is True else (frontend or None)
self.ettin = "ettin_ids" in [i.name for i in self.session.get_inputs()] # kale-tts-1-tiny's Ettin text side
if self.ettin:
if not isinstance(self.frontend, QuickJSFrontEnd): raise RuntimeError("this model's Ettin text side needs the QuickJS front end (pip install quickjs)")
self.frontend.init_ettin((d / "ettin-tokenizer.json").read_text())
self.sample_rate = SAMPLE_RATE
def list_voices(self):
return [{"id": v["id"], "name": v.get("name", v["id"]), "description": v.get("description"), "accent": v.get("accent", "en-us"),
"label": f"{v['name']} · {v['description']}" if v.get("description") else v.get("name", v["id"])} for v in self.voices]
def resolve_voice(self, voice):
"""a voice by its public name (any case, e.g. "kenzie") or its internal id ("s057b") -> the id"""
k = str(voice).strip().lower()
for v in self.voices:
if v["id"].lower() == k or v.get("name", "").lower() == k or k in v.get("aliases", []): return v["id"] # aliases: earlier names
raise ValueError(f"unknown voice {voice}; voices: {', '.join(v.get('name', v['id']).lower() + ' (' + v['id'] + ')' for v in self.voices)}")
def voice(self, vid=None):
vid = self.resolve_voice(vid if vid is not None else self.voices[0]["id"])
if vid not in self._v:
meta = next(v for v in self.voices if v["id"] == vid)
sb, eb = (self.dir / f"voices/{vid}.bin").stat().st_size, (self.dir / f"voices/{vid}.extra.bin").stat().st_size
need = sum(int(np.prod(vi["dims"])) for vi in self.voice_inputs) * 4
if sb == 0 or sb % (256 * 4): raise ValueError(f"voice {vid}: voices/{vid}.bin is {sb} bytes, not a style table of 256-float rows (wrong file?)")
if eb != need: raise ValueError(f"voice {vid}: voices/{vid}.extra.bin is {eb} bytes, voices.json's voice_inputs need {need} (wrong file?)")
st = np.fromfile(self.dir / f"voices/{vid}.bin", np.float32).reshape(-1, 256)
ex, o, feeds = np.fromfile(self.dir / f"voices/{vid}.extra.bin", np.float32), 0, {}
for vi in self.voice_inputs:
n = int(np.prod(vi["dims"])); feeds[vi["name"]] = ex[o:o + n].reshape(vi["dims"]); o += n
self._v[vid] = (meta, st, feeds)
return self._v[vid]
def speak_ids(self, ids, voice=None, ettin=None):
"""one sentence of Kokoro-table ids (no pads) -> float32 audio. ettin: this sentence's Ettin inputs (QuickJSFrontEnd.ettin), for
models with the Ettin text side"""
_, st, feeds = self.voice(voice); ids = list(ids)[:510]
x = np.array([[0, *ids, 0]], np.int64)
f = {"input_ids": x, "style": st[min(len(ids), len(st)) - 1][None], "speed": np.ones(1, np.float32), **feeds}
if self.ettin:
if ettin is None: raise ValueError("this model needs the sentence's Ettin inputs: use speak() / stream()")
w = ettin["widx"][:len(ids)]
f.update(ettin_ids=np.array([ettin["ids"]], np.int64), ettin_mask=np.ones((1, ettin["S"]), np.int64),
wavg=np.array(ettin["wavg"], np.float32).reshape(1, ettin["W"], ettin["S"]), widx=np.array([[w[0], *w, w[-1]]], np.int64))
return self.session.run(None, f)[0][0]
def _ids(self, ipa, accent="en-us"):
chars = [c for c in ipa if c in self.vocab]
return [self.vocab[c] for c in (kn_ipa(chars, accent == "en-gb") if self.kn else chars) if c in self.vocab]
def speak_ipa(self, ipa, voice=None): return self.speak_ids(self._ids(ipa, self.voice(voice)[0].get("accent", "en-us")), voice)
def stream(self, text, voice=None):
"""sentence by sentence: yields float32 audio as each sentence is ready"""
if not self.frontend: raise RuntimeError("no front end: use speak_ids()/speak_ipa()")
meta = self.voice(voice)[0]; acc = meta.get("accent", "en-us"); prev = ""
for s in self.frontend(text, acc):
ids = self._ids(s["ipa"], acc)
if not ids: continue
e = None
if self.ettin:
per = [self._ids(w, acc) for w in self.frontend.ipa_words(s["text"], acc)]
e = self.frontend.ettin(s["text"], prev, ids, per)
yield self.speak_ids(ids, voice, e); prev = s["text"]
def speak(self, text, voice=None, gap_s=0.12):
gap = np.zeros(int(gap_s * SAMPLE_RATE), np.float32); parts = []
for a in self.stream(text, voice): parts += [a, gap]
return np.concatenate(parts) if parts else np.zeros(0, np.float32)
def to_wav(audio, sr=SAMPLE_RATE):
"""16-bit PCM WAV bytes"""
import io, wave
b = io.BytesIO(); pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2")
with wave.open(b, "wb") as w: w.setnchannels(1); w.setsampwidth(2); w.setframerate(sr); w.writeframes(pcm.tobytes())
return b.getvalue()
def to_pcm16(audio): return (np.clip(audio, -1, 1) * 32767).astype("<i2").tobytes()