apptor-tiny-tts-at / speak.py
apptor's picture
apptor-tiny-tts-at v0.1: Austrian accent switch, no eSpeak, browser demo
48732a6 verified
Raw History Blame Contribute Delete
6.6 kB
"""apptor-tiny-tts: German text -> WAV on the CPU. No eSpeak NG, no torch, no GPU.
pip install onnxruntime numpy num2words
python speak.py "Servus, wie geht's dir?" --voice M2 --out servus.wav
python speak.py "Servus, wie geht's dir?" --accent 0 # Austrian model: Hochdeutsch
python speak.py "Servus, wie geht's dir?" --accent 1.6 # Austrian model: strongest usable accent
Uses the files next to this script: the speech model (apptor-tiny-tts-at.onnx, else apptor-tiny-tts-de.onnx), the
letter-to-sound model apptor-g2p-de.onnx and, unless --no-vocos, the sound filter apptor-vocos-de.int8.onnx.
"""
import argparse
import json
import os
import re
import wave
from pathlib import Path
import numpy as np
import onnxruntime as ort
HERE = Path(__file__).resolve().parent
os.environ.setdefault("APPTOR_G2P_MODEL", str(HERE / "apptor-g2p-de.onnx"))
from tinyvoca.g2p_de import phonemize # noqa: E402
def session(path, threads):
options = ort.SessionOptions()
options.intra_op_num_threads = threads
return ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"])
def vocos_kernels(n_fft=1024):
"""Fixed STFT / inverse STFT kernels of the Vocos ONNX model (periodic Hann window)."""
n = np.arange(n_fft)
k = np.arange(n_fft // 2 + 1)[:, None]
w = 0.5 - 0.5 * np.cos(2 * np.pi * n / n_fft)
ang = 2 * np.pi * k * n / n_fft
c = np.where((k == 0) | (k == n_fft // 2), 1.0, 2.0) / n_fft
return {"a_real": (w * np.cos(ang))[:, None].astype(np.float32),
"a_imag": (-w * np.sin(ang))[:, None].astype(np.float32),
"synth": np.concatenate((c * np.cos(ang) * w, -c * np.sin(ang) * w))[:, None].astype(np.float32)}
def trim_tail(audio, sr, floor_db=-30.0, fade_ms=40.0):
"""Cut the breathy noise after the last sound: keep up to the last 10 ms frame within 30 dB of the loudest one,
then fade out over 40 ms."""
hop = sr // 100
n = len(audio) // hop
if n == 0:
return audio
e = (audio[:n * hop].reshape(n, hop) ** 2).sum(1)
if e.max() <= 0:
return audio
last = int(np.nonzero(e >= e.max() * 10 ** (floor_db / 10))[0][-1])
fade = int(fade_ms * sr / 1000)
end = (last + 1) * hop + fade
if end >= len(audio):
return audio
out = audio[:end].copy()
out[-fade:] *= 0.5 + 0.5 * np.cos(np.pi * np.arange(1, fade + 1) / fade)
return out
class Speaker:
def __init__(self, model=None, vocos=True, threads=4):
model = Path(model) if model else next(p for p in (HERE / "apptor-tiny-tts-at.onnx", HERE / "apptor-tiny-tts-de.onnx")
if p.is_file())
self.meta = json.loads(model.with_suffix(".json").read_text(encoding="utf-8"))
self.tts = session(model, threads)
self.inputs = {i.name for i in self.tts.get_inputs()}
self.max_tokens = self.meta.get("max_tokens", 512)
self.vocos = session(HERE / "apptor-vocos-de.int8.onnx", threads) if vocos else None
self.kernels = vocos_kernels() if vocos else None
@property
def voices(self):
return [v for v in self.meta["voices"] if v != "AT"]
def _pieces(self, text, austrian):
"""Sentences, then clauses or halves, so that no piece exceeds the model's token limit."""
for sentence in re.split(r"(?<=[.!?…])\s+", text.strip()):
if not sentence:
continue
p = phonemize(sentence, austrian=austrian)
if len(p.ids) <= self.max_tokens:
yield p
continue
parts = re.split(r"(?<=[,;:—])\s+", sentence)
if len(parts) == 1:
words = sentence.split()
parts = [" ".join(words[:len(words) // 2]), " ".join(words[len(words) // 2:])]
for part in parts:
yield from self._pieces(part, austrian)
def speak(self, text, voice="M2", accent=1.0, speed=1.0, gap=0.12):
"""-> float32 audio at meta["sample_rate"]. accent: 0 = Hochdeutsch, 1.0 = Austrian, up to 1.6 (AT model only)."""
sr = self.meta["sample_rate"]
has_accent = "accent" in self.inputs
out = []
for p in self._pieces(text, austrian=has_accent and accent >= 0.5):
feeds = {"tokens": np.array([p.ids], np.int64), "stress": np.array([p.stress], np.int64),
"voice": np.array([self.meta["voices"].index(voice)], np.int64),
"speed": np.array([speed], np.float32), "pitch_shift": np.array([0.0], np.float32),
"pitch_range": np.array([1.0], np.float32)}
if has_accent:
feeds["accent"] = np.array([accent], np.float32)
audio = self.tts.run(["audio"], feeds)[0][0].astype(np.float32)
if self.vocos is not None:
y = self.vocos.run(None, {"audio": audio[None], **self.kernels})[0][0]
audio = y[:len(audio)]
audio = trim_tail(audio, sr)
peak = float(np.abs(audio).max()) if len(audio) else 0.0
if peak > 0.95:
audio = audio * (0.95 / peak)
if out:
out.append(np.zeros(int(gap * sr), np.float32))
out.append(audio)
return np.concatenate(out) if out else np.zeros(0, np.float32)
def write_wav(path, audio, sr):
"""float audio -> 16-bit mono WAV."""
with wave.open(str(path), "wb") as w:
w.setnchannels(1)
w.setsampwidth(2)
w.setframerate(sr)
w.writeframes((np.clip(audio, -1, 1) * 32767).astype("<i2").tobytes())
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("text")
ap.add_argument("--voice", default="M2", choices=["M2", "F2"])
ap.add_argument("--accent", type=float, default=1.0, help="Austrian model: 0 Hochdeutsch, 1.0 Austrian, up to 1.6")
ap.add_argument("--speed", type=float, default=1.0)
ap.add_argument("--out", default="out.wav")
ap.add_argument("--model", help="speech model .onnx (default: the one next to this script)")
ap.add_argument("--no-vocos", action="store_true", help="skip the sound filter (faster, more buzzy)")
ap.add_argument("--threads", type=int, default=4)
a = ap.parse_args()
s = Speaker(a.model, vocos=not a.no_vocos, threads=a.threads)
audio = s.speak(a.text, voice=a.voice, accent=a.accent, speed=a.speed)
write_wav(a.out, audio, s.meta["sample_rate"])
print(f"wrote {a.out} ({len(audio) / s.meta['sample_rate']:.1f} s)")
if __name__ == "__main__":
main()