Download python/kale_tts.py from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/python/kale_tts.py
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/python/kale_tts.py
-
curl -L -o kale_tts.py https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/python/kale_tts.py
12.2 kB
| """kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs, | |
| kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself. | |
| pip install onnxruntime numpy quickjs (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name) | |
| from kale_tts import KaleTTS | |
| tts = KaleTTS("models/kale-tts-1-nano") | |
| audio = tts.speak("A volcano is a place where very hot, melted rock comes up out of the ground.", voice="kenzie") # float32, 24 kHz | |
| for chunk in tts.stream(long_text, voice="d03"): play(chunk) # sentence by sentence | |
| """ | |
| import json, os, shutil, subprocess, threading | |
| from pathlib import Path | |
| import numpy as np | |
| import onnxruntime as ort | |
| SAMPLE_RATE = 24000 | |
| EN_GB_ON = True # (16 Nov: the 10-voice files are trained on en-gb ids for d03 / d13) British front end for en-gb voices (d03, d13); on only with weights trained on en-gb ids (= js/kale-tts.mjs EN_GB_ON) | |
| HERE = Path(__file__).resolve().parent | |
| def _near(name): | |
| """the release's frontend/ or models/ folder: next to this file (a host's own copy), or at the repo root (python/ sits one level | |
| below it, runtimes/python/ two), so both copies of this package are the same file""" | |
| for d in (HERE, *list(HERE.parents)[:2]): | |
| if (d / name).is_dir(): return d / name | |
| return HERE.parent / name | |
| class FrontEnd: | |
| """text -> sentences -> ids, via one long-lived `node frontend_cli.mjs` (thread-safe).""" | |
| def __init__(self, cli=HERE / "frontend_cli.mjs", node=None): | |
| node = node or os.environ.get("KALE_NODE") or shutil.which("node") | |
| if not node: raise RuntimeError("the text front end needs node (>= 18); or call speak_ids()/speak_ipa()") | |
| self.p = subprocess.Popen([node, str(cli)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1) | |
| self.lock = threading.Lock() | |
| def __call__(self, text, accent="en-us"): | |
| with self.lock: | |
| self.p.stdin.write(json.dumps({"text": text, "accent": accent}) + "\n"); self.p.stdin.flush(); r = json.loads(self.p.stdout.readline()) | |
| if "error" in r: raise ValueError(r["error"]) | |
| return r["sentences"] | |
| def close(self): self.p.terminate() | |
| class QuickJSFrontEnd: | |
| """the same front end without node: the bundled script (kale_frontend.js, built from the release front end) in QuickJS | |
| (pip install quickjs). Same output as FrontEnd. Thread-safe (one lock).""" | |
| def __init__(self, frontend_dir=None, bundle=HERE / "kale_frontend.js"): | |
| import base64, quickjs | |
| fd = Path(frontend_dir) if frontend_dir else _near("frontend") | |
| m = fd / "words"; self.ctx = quickjs.Context(); self.ctx.set_memory_limit(1 << 30); self.ctx.eval(Path(bundle).read_text()) | |
| self.ctx.eval("globalThis.kaleInit = (...a) => kaleFrontEnd.init(...a); globalThis.kaleInitGb = (...a) => kaleFrontEnd.initGb(...a);" | |
| "globalThis.kaleRun = (t, a) => kaleFrontEnd.run(t, a);") | |
| files = lambda d: [*[(d / f).read_text() for f in ("lexicon.tsv", "override.public.tsv", "terms.tsv")], # noqa: E731 | |
| base64.b64encode((d / "g2p.bin").read_bytes()).decode()] | |
| self.ctx.get("kaleInit")(*files(m)) | |
| self.accents = {"en-us"} | |
| if EN_GB_ON and (m / "en-gb/g2p.bin").exists(): # the British front end, as the JS API's EN_GB_ON | |
| self.ctx.get("kaleInitGb")(*files(m / "en-gb")); self.accents.add("en-gb") | |
| self.run = self.ctx.get("kaleRun"); self.lock = threading.Lock() | |
| self.ctx.eval("globalThis.kaleIpaWords = (t, a) => kaleFrontEnd.ipaWords(t, a); globalThis.kaleInitEttin = (j) => kaleFrontEnd.initEttin(j);" | |
| "globalThis.kaleEttin = (t, p, s, w) => kaleFrontEnd.ettin(t, p, s, w);") | |
| self.ettin_ready = False | |
| def init_ettin(self, tokenizer_json): | |
| with self.lock: self.ctx.get("kaleInitEttin")(tokenizer_json); self.ettin_ready = True | |
| def ipa_words(self, text, accent="en-us"): | |
| with self.lock: return json.loads(self.ctx.get("kaleIpaWords")(text, accent if accent in self.accents else "en-us")) | |
| def ettin(self, text, prev, sent_ids, word_ids): | |
| with self.lock: return json.loads(self.ctx.get("kaleEttin")(text, prev, json.dumps(sent_ids), json.dumps(word_ids))) | |
| def __call__(self, text, accent="en-us"): | |
| """-> [{text, ipa}] one per sentence; accents without a front end use en-us""" | |
| with self.lock: return json.loads(self.run(text, accent if accent in self.accents else "en-us")) | |
| def close(self): pass | |
| # Kokoro-native phoneme ids (voices.json "phonemes": "kokoro-native", the 17 Nov kale-tts-1-tiny) = js/kale-tts.mjs knIPA, line for line. | |
| _KN_PAIRS = {"eɪ": "A", "aɪ": "I", "aʊ": "W", "ɔɪ": "Y", "tʃ": "ʧ", "dʒ": "ʤ"} | |
| _KN_VOW = set("aeiouɑɐɒæəɘɚɛɜɝɞɪɨʊʉʌɔøœɤɯʏyAIWOYᵊᵻ"); _KN_END = set(" .,!?;:") | |
| def kn_ipa(chars, gb): | |
| """chars: the IPA characters that have ids -> the Kokoro-native characters (en-us rules, or the British convention when gb)""" | |
| pairs = {**_KN_PAIRS, **({"əʊ": "Q"} if gb else {"oʊ": "O"})}; out = []; i = 0 | |
| while i < len(chars): | |
| two = chars[i] + (chars[i + 1] if i + 1 < len(chars) else "") | |
| if two in pairs: out.append(pairs[two]); i += 2; continue | |
| c = chars[i]; i += 1 | |
| if gb: out.append(c); continue | |
| if c == "ː": continue | |
| out += ["ə", "ɹ"] if c == "ɚ" else ["ɜ", "ɹ"] if c == "ɝ" else ["T"] if c == "ɾ" else [c] | |
| for k in range(1, len(out) - 1): | |
| if out[k] == "ə" and out[k + 1] in "lnm" and (k + 2 == len(out) or out[k + 2] in _KN_END) and out[k - 1] not in _KN_VOW \ | |
| and out[k - 1] not in "ˈˌ" and not out[k - 1].isspace(): | |
| out[k] = "ᵊ" | |
| return out | |
| def default_frontend(): | |
| """QuickJS when installed (no node needed), else the node helper""" | |
| try: | |
| import quickjs # noqa: F401 | |
| return QuickJSFrontEnd() | |
| except ImportError: | |
| return FrontEnd() | |
| class KaleTTS: | |
| def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None): | |
| d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir) | |
| vj = json.loads((d / "voices.json").read_text()) | |
| model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph) | |
| so = ort.SessionOptions() | |
| if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1 | |
| self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers)) | |
| self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"] | |
| self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {} | |
| self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa) | |
| self.frontend = default_frontend() if frontend is True else (frontend or None) | |
| self.ettin = "ettin_ids" in [i.name for i in self.session.get_inputs()] # kale-tts-1-tiny's Ettin text side | |
| if self.ettin: | |
| if not isinstance(self.frontend, QuickJSFrontEnd): raise RuntimeError("this model's Ettin text side needs the QuickJS front end (pip install quickjs)") | |
| self.frontend.init_ettin((d / "ettin-tokenizer.json").read_text()) | |
| self.sample_rate = SAMPLE_RATE | |
| def list_voices(self): | |
| return [{"id": v["id"], "name": v.get("name", v["id"]), "description": v.get("description"), "accent": v.get("accent", "en-us"), | |
| "label": f"{v['name']} · {v['description']}" if v.get("description") else v.get("name", v["id"])} for v in self.voices] | |
| def resolve_voice(self, voice): | |
| """a voice by its public name (any case, e.g. "kenzie") or its internal id ("s057b") -> the id""" | |
| k = str(voice).strip().lower() | |
| for v in self.voices: | |
| if v["id"].lower() == k or v.get("name", "").lower() == k or k in v.get("aliases", []): return v["id"] # aliases: earlier names | |
| raise ValueError(f"unknown voice {voice}; voices: {', '.join(v.get('name', v['id']).lower() + ' (' + v['id'] + ')' for v in self.voices)}") | |
| def voice(self, vid=None): | |
| vid = self.resolve_voice(vid if vid is not None else self.voices[0]["id"]) | |
| if vid not in self._v: | |
| meta = next(v for v in self.voices if v["id"] == vid) | |
| sb, eb = (self.dir / f"voices/{vid}.bin").stat().st_size, (self.dir / f"voices/{vid}.extra.bin").stat().st_size | |
| need = sum(int(np.prod(vi["dims"])) for vi in self.voice_inputs) * 4 | |
| if sb == 0 or sb % (256 * 4): raise ValueError(f"voice {vid}: voices/{vid}.bin is {sb} bytes, not a style table of 256-float rows (wrong file?)") | |
| if eb != need: raise ValueError(f"voice {vid}: voices/{vid}.extra.bin is {eb} bytes, voices.json's voice_inputs need {need} (wrong file?)") | |
| st = np.fromfile(self.dir / f"voices/{vid}.bin", np.float32).reshape(-1, 256) | |
| ex, o, feeds = np.fromfile(self.dir / f"voices/{vid}.extra.bin", np.float32), 0, {} | |
| for vi in self.voice_inputs: | |
| n = int(np.prod(vi["dims"])); feeds[vi["name"]] = ex[o:o + n].reshape(vi["dims"]); o += n | |
| self._v[vid] = (meta, st, feeds) | |
| return self._v[vid] | |
| def speak_ids(self, ids, voice=None, ettin=None): | |
| """one sentence of Kokoro-table ids (no pads) -> float32 audio. ettin: this sentence's Ettin inputs (QuickJSFrontEnd.ettin), for | |
| models with the Ettin text side""" | |
| _, st, feeds = self.voice(voice); ids = list(ids)[:510] | |
| x = np.array([[0, *ids, 0]], np.int64) | |
| f = {"input_ids": x, "style": st[min(len(ids), len(st)) - 1][None], "speed": np.ones(1, np.float32), **feeds} | |
| if self.ettin: | |
| if ettin is None: raise ValueError("this model needs the sentence's Ettin inputs: use speak() / stream()") | |
| w = ettin["widx"][:len(ids)] | |
| f.update(ettin_ids=np.array([ettin["ids"]], np.int64), ettin_mask=np.ones((1, ettin["S"]), np.int64), | |
| wavg=np.array(ettin["wavg"], np.float32).reshape(1, ettin["W"], ettin["S"]), widx=np.array([[w[0], *w, w[-1]]], np.int64)) | |
| return self.session.run(None, f)[0][0] | |
| def _ids(self, ipa, accent="en-us"): | |
| chars = [c for c in ipa if c in self.vocab] | |
| return [self.vocab[c] for c in (kn_ipa(chars, accent == "en-gb") if self.kn else chars) if c in self.vocab] | |
| def speak_ipa(self, ipa, voice=None): return self.speak_ids(self._ids(ipa, self.voice(voice)[0].get("accent", "en-us")), voice) | |
| def stream(self, text, voice=None): | |
| """sentence by sentence: yields float32 audio as each sentence is ready""" | |
| if not self.frontend: raise RuntimeError("no front end: use speak_ids()/speak_ipa()") | |
| meta = self.voice(voice)[0]; acc = meta.get("accent", "en-us"); prev = "" | |
| for s in self.frontend(text, acc): | |
| ids = self._ids(s["ipa"], acc) | |
| if not ids: continue | |
| e = None | |
| if self.ettin: | |
| per = [self._ids(w, acc) for w in self.frontend.ipa_words(s["text"], acc)] | |
| e = self.frontend.ettin(s["text"], prev, ids, per) | |
| yield self.speak_ids(ids, voice, e); prev = s["text"] | |
| def speak(self, text, voice=None, gap_s=0.12): | |
| gap = np.zeros(int(gap_s * SAMPLE_RATE), np.float32); parts = [] | |
| for a in self.stream(text, voice): parts += [a, gap] | |
| return np.concatenate(parts) if parts else np.zeros(0, np.float32) | |
| def to_wav(audio, sr=SAMPLE_RATE): | |
| """16-bit PCM WAV bytes""" | |
| import io, wave | |
| b = io.BytesIO(); pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2") | |
| with wave.open(b, "wb") as w: w.setnchannels(1); w.setsampwidth(2); w.setframerate(sr); w.writeframes(pcm.tobytes()) | |
| return b.getvalue() | |
| def to_pcm16(audio): return (np.clip(audio, -1, 1) * 32767).astype("<i2").tobytes() | |