File size: 12,157 Bytes
f21d6eb 6901100 f21d6eb 6901100 f21d6eb 6901100 f21d6eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | """kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
pip install onnxruntime numpy quickjs (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name)
from kale_tts import KaleTTS
tts = KaleTTS("models/kale-tts-1-nano")
audio = tts.speak("A volcano is a place where very hot, melted rock comes up out of the ground.", voice="kenzie") # float32, 24 kHz
for chunk in tts.stream(long_text, voice="d03"): play(chunk) # sentence by sentence
"""
import json, os, shutil, subprocess, threading
from pathlib import Path
import numpy as np
import onnxruntime as ort
SAMPLE_RATE = 24000
EN_GB_ON = True # (16 Nov: the 10-voice files are trained on en-gb ids for d03 / d13) British front end for en-gb voices (d03, d13); on only with weights trained on en-gb ids (= js/kale-tts.mjs EN_GB_ON)
HERE = Path(__file__).resolve().parent
def _near(name):
"""the release's frontend/ or models/ folder: next to this file (a host's own copy), or at the repo root (python/ sits one level
below it, runtimes/python/ two), so both copies of this package are the same file"""
for d in (HERE, *list(HERE.parents)[:2]):
if (d / name).is_dir(): return d / name
return HERE.parent / name
class FrontEnd:
"""text -> sentences -> ids, via one long-lived `node frontend_cli.mjs` (thread-safe)."""
def __init__(self, cli=HERE / "frontend_cli.mjs", node=None):
node = node or os.environ.get("KALE_NODE") or shutil.which("node")
if not node: raise RuntimeError("the text front end needs node (>= 18); or call speak_ids()/speak_ipa()")
self.p = subprocess.Popen([node, str(cli)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1)
self.lock = threading.Lock()
def __call__(self, text, accent="en-us"):
with self.lock:
self.p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); self.p.stdin.flush(); r = json.loads(self.p.stdout.readline())
if "error" in r: raise ValueError(r["error"])
return r["sentences"]
def close(self): self.p.terminate()
class QuickJSFrontEnd:
"""the same front end without node: the bundled script (kale_frontend.js, built from the release front end) in QuickJS
(pip install quickjs). Same output as FrontEnd. Thread-safe (one lock)."""
def __init__(self, frontend_dir=None, bundle=HERE / "kale_frontend.js"):
import base64, quickjs
fd = Path(frontend_dir) if frontend_dir else _near("frontend")
m = fd / "words"; self.ctx = quickjs.Context(); self.ctx.set_memory_limit(1 << 30); self.ctx.eval(Path(bundle).read_text())
self.ctx.eval("globalThis.kaleInit = (...a) => kaleFrontEnd.init(...a); globalThis.kaleInitGb = (...a) => kaleFrontEnd.initGb(...a);"
"globalThis.kaleRun = (t, a) => kaleFrontEnd.run(t, a);")
files = lambda d: [*[(d / f).read_text() for f in ("lexicon.tsv", "override.public.tsv", "terms.tsv")], # noqa: E731
base64.b64encode((d / "g2p.bin").read_bytes()).decode()]
self.ctx.get("kaleInit")(*files(m))
self.accents = {"en-us"}
if EN_GB_ON and (m / "en-gb/g2p.bin").exists(): # the British front end, as the JS API's EN_GB_ON
self.ctx.get("kaleInitGb")(*files(m / "en-gb")); self.accents.add("en-gb")
self.run = self.ctx.get("kaleRun"); self.lock = threading.Lock()
self.ctx.eval("globalThis.kaleIpaWords = (t, a) => kaleFrontEnd.ipaWords(t, a); globalThis.kaleInitEttin = (j) => kaleFrontEnd.initEttin(j);"
"globalThis.kaleEttin = (t, p, s, w) => kaleFrontEnd.ettin(t, p, s, w);")
self.ettin_ready = False
def init_ettin(self, tokenizer_json):
with self.lock: self.ctx.get("kaleInitEttin")(tokenizer_json); self.ettin_ready = True
def ipa_words(self, text, accent="en-us"):
with self.lock: return json.loads(self.ctx.get("kaleIpaWords")(text, accent if accent in self.accents else "en-us"))
def ettin(self, text, prev, sent_ids, word_ids):
with self.lock: return json.loads(self.ctx.get("kaleEttin")(text, prev, json.dumps(sent_ids), json.dumps(word_ids)))
def __call__(self, text, accent="en-us"):
"""-> [{text, ipa}] one per sentence; accents without a front end use en-us"""
with self.lock: return json.loads(self.run(text, accent if accent in self.accents else "en-us"))
def close(self): pass
# Kokoro-native phoneme ids (voices.json "phonemes": "kokoro-native", the 17 Nov kale-tts-1-tiny) = js/kale-tts.mjs knIPA, line for line.
_KN_PAIRS = {"eɪ": "A", "aɪ": "I", "aʊ": "W", "ɔɪ": "Y", "tʃ": "ʧ", "dʒ": "ʤ"}
_KN_VOW = set("aeiouɑɐɒæəɘɚɛɜɝɞɪɨʊʉʌɔøœɤɯʏyAIWOYᵊᵻ"); _KN_END = set(" .,!?;:")
def kn_ipa(chars, gb):
"""chars: the IPA characters that have ids -> the Kokoro-native characters (en-us rules, or the British convention when gb)"""
pairs = {**_KN_PAIRS, **({"əʊ": "Q"} if gb else {"oʊ": "O"})}; out = []; i = 0
while i < len(chars):
two = chars[i] + (chars[i + 1] if i + 1 < len(chars) else "")
if two in pairs: out.append(pairs[two]); i += 2; continue
c = chars[i]; i += 1
if gb: out.append(c); continue
if c == "ː": continue
out += ["ə", "ɹ"] if c == "ɚ" else ["ɜ", "ɹ"] if c == "ɝ" else ["T"] if c == "ɾ" else [c]
for k in range(1, len(out) - 1):
if out[k] == "ə" and out[k + 1] in "lnm" and (k + 2 == len(out) or out[k + 2] in _KN_END) and out[k - 1] not in _KN_VOW \
and out[k - 1] not in "ˈˌ" and not out[k - 1].isspace():
out[k] = "ᵊ"
return out
def default_frontend():
"""QuickJS when installed (no node needed), else the node helper"""
try:
import quickjs # noqa: F401
return QuickJSFrontEnd()
except ImportError:
return FrontEnd()
class KaleTTS:
def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
vj = json.loads((d / "voices.json").read_text())
model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph)
so = ort.SessionOptions()
if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"]
self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa)
self.frontend = default_frontend() if frontend is True else (frontend or None)
self.ettin = "ettin_ids" in [i.name for i in self.session.get_inputs()] # kale-tts-1-tiny's Ettin text side
if self.ettin:
if not isinstance(self.frontend, QuickJSFrontEnd): raise RuntimeError("this model's Ettin text side needs the QuickJS front end (pip install quickjs)")
self.frontend.init_ettin((d / "ettin-tokenizer.json").read_text())
self.sample_rate = SAMPLE_RATE
def list_voices(self):
return [{"id": v["id"], "name": v.get("name", v["id"]), "description": v.get("description"), "accent": v.get("accent", "en-us"),
"label": f"{v['name']} · {v['description']}" if v.get("description") else v.get("name", v["id"])} for v in self.voices]
def resolve_voice(self, voice):
"""a voice by its public name (any case, e.g. "kenzie") or its internal id ("s057b") -> the id"""
k = str(voice).strip().lower()
for v in self.voices:
if v["id"].lower() == k or v.get("name", "").lower() == k or k in v.get("aliases", []): return v["id"] # aliases: earlier names
raise ValueError(f"unknown voice {voice}; voices: {', '.join(v.get('name', v['id']).lower() + ' (' + v['id'] + ')' for v in self.voices)}")
def voice(self, vid=None):
vid = self.resolve_voice(vid if vid is not None else self.voices[0]["id"])
if vid not in self._v:
meta = next(v for v in self.voices if v["id"] == vid)
sb, eb = (self.dir / f"voices/{vid}.bin").stat().st_size, (self.dir / f"voices/{vid}.extra.bin").stat().st_size
need = sum(int(np.prod(vi["dims"])) for vi in self.voice_inputs) * 4
if sb == 0 or sb % (256 * 4): raise ValueError(f"voice {vid}: voices/{vid}.bin is {sb} bytes, not a style table of 256-float rows (wrong file?)")
if eb != need: raise ValueError(f"voice {vid}: voices/{vid}.extra.bin is {eb} bytes, voices.json's voice_inputs need {need} (wrong file?)")
st = np.fromfile(self.dir / f"voices/{vid}.bin", np.float32).reshape(-1, 256)
ex, o, feeds = np.fromfile(self.dir / f"voices/{vid}.extra.bin", np.float32), 0, {}
for vi in self.voice_inputs:
n = int(np.prod(vi["dims"])); feeds[vi["name"]] = ex[o:o + n].reshape(vi["dims"]); o += n
self._v[vid] = (meta, st, feeds)
return self._v[vid]
def speak_ids(self, ids, voice=None, ettin=None):
"""one sentence of Kokoro-table ids (no pads) -> float32 audio. ettin: this sentence's Ettin inputs (QuickJSFrontEnd.ettin), for
models with the Ettin text side"""
_, st, feeds = self.voice(voice); ids = list(ids)[:510]
x = np.array([[0, *ids, 0]], np.int64)
f = {"input_ids": x, "style": st[min(len(ids), len(st)) - 1][None], "speed": np.ones(1, np.float32), **feeds}
if self.ettin:
if ettin is None: raise ValueError("this model needs the sentence's Ettin inputs: use speak() / stream()")
w = ettin["widx"][:len(ids)]
f.update(ettin_ids=np.array([ettin["ids"]], np.int64), ettin_mask=np.ones((1, ettin["S"]), np.int64),
wavg=np.array(ettin["wavg"], np.float32).reshape(1, ettin["W"], ettin["S"]), widx=np.array([[w[0], *w, w[-1]]], np.int64))
return self.session.run(None, f)[0][0]
def _ids(self, ipa, accent="en-us"):
chars = [c for c in ipa if c in self.vocab]
return [self.vocab[c] for c in (kn_ipa(chars, accent == "en-gb") if self.kn else chars) if c in self.vocab]
def speak_ipa(self, ipa, voice=None): return self.speak_ids(self._ids(ipa, self.voice(voice)[0].get("accent", "en-us")), voice)
def stream(self, text, voice=None):
"""sentence by sentence: yields float32 audio as each sentence is ready"""
if not self.frontend: raise RuntimeError("no front end: use speak_ids()/speak_ipa()")
meta = self.voice(voice)[0]; acc = meta.get("accent", "en-us"); prev = ""
for s in self.frontend(text, acc):
ids = self._ids(s["ipa"], acc)
if not ids: continue
e = None
if self.ettin:
per = [self._ids(w, acc) for w in self.frontend.ipa_words(s["text"], acc)]
e = self.frontend.ettin(s["text"], prev, ids, per)
yield self.speak_ids(ids, voice, e); prev = s["text"]
def speak(self, text, voice=None, gap_s=0.12):
gap = np.zeros(int(gap_s * SAMPLE_RATE), np.float32); parts = []
for a in self.stream(text, voice): parts += [a, gap]
return np.concatenate(parts) if parts else np.zeros(0, np.float32)
def to_wav(audio, sr=SAMPLE_RATE):
"""16-bit PCM WAV bytes"""
import io, wave
b = io.BytesIO(); pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2")
with wave.open(b, "wb") as w: w.setnchannels(1); w.setsampwidth(2); w.setframerate(sr); w.writeframes(pcm.tobytes())
return b.getvalue()
def to_pcm16(audio): return (np.clip(audio, -1, 1) * 32767).astype("<i2").tobytes()
|