HY-2012's picture
Upload folder using huggingface_hub
f683e5c verified
Raw History Blame Contribute Delete
28 kB
"""M5-TTS AX650 runtime (numpy + axengine; no torch / onnxruntime needed).
reference wav --codec_encoder--> prompt codes --host RVQ--> prompt embeds
text ids + prompt embeds --lm_prefill--> hidden + KV
loop per frame (12.5 fps):
hidden --head0/sample--> c0
[hidden, c0] --depth_prefill--> h + KV
h --audio_head[i]/sample--> c_i --depth_step (x14)--> next h
sum(audio_emb[codes]) --lm_decode--> next hidden
prompt+generated codes --host RVQ--> latent --codec_decoder--> 24 kHz wav
"""
from __future__ import annotations
import json
import math
import time
from pathlib import Path
import numpy as np
NEG = -30.0
# ------------------------------------------------------------------ engines
class OnnxEngine:
def __init__(self, path):
import onnxruntime as ort
self.sess = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"])
self.input_names = [i.name for i in self.sess.get_inputs()]
self.output_names = [o.name for o in self.sess.get_outputs()]
def run(self, feed):
return self.sess.run(self.output_names, {k: np.ascontiguousarray(feed[k]) for k in self.input_names})
class AxEngine:
def __init__(self, path):
import axengine as axe
self.sess = axe.InferenceSession(str(path), providers=["AxEngineExecutionProvider"])
self.input_names = [i.name for i in self.sess.get_inputs()]
self.output_names = [o.name for o in self.sess.get_outputs()]
def run(self, feed):
return self.sess.run(self.output_names, {k: np.ascontiguousarray(feed[k], np.float32) for k in self.input_names})
# ------------------------------------------------------------------ helpers
def read_wav(path, sample_rate=24000):
import soundfile as sf
from math import gcd
audio, sr = sf.read(str(path), dtype="float32")
if audio.ndim == 2:
audio = audio.mean(axis=1)
if sr != sample_rate:
from scipy.signal import resample_poly
g = gcd(sr, sample_rate)
audio = resample_poly(audio, sample_rate // g, sr // g).astype(np.float32)
audio = audio / (np.max(np.abs(audio)) + 1e-8)
threshold = 0.02 if len(audio) > 10 * sample_rate else 0.015
speech = np.flatnonzero(np.abs(audio) > threshold)
if len(speech):
cuts = np.flatnonzero(np.diff(speech) > int(0.15 * sample_rate))
starts = np.r_[speech[0], speech[cuts + 1]]
final_end = speech[-1] + 1 if len(audio) - speech[-1] - 1 >= int(0.15 * sample_rate) else len(audio)
ends = np.r_[speech[cuts] + 1, final_end]
audio = np.concatenate([audio[max(0, a - 1200):min(len(audio), b + 1200)]
for a, b in zip(starts, ends)])
return np.ascontiguousarray(audio, np.float32)
def write_wav(path, wav, sample_rate=24000):
import soundfile as sf
sf.write(str(path), np.clip(wav, -1, 1).astype(np.float32), int(sample_rate),
subtype="PCM_16", format="WAV")
def rope_tables(inv_freq, scale, positions):
freqs = np.asarray(positions, np.float32)[:, None] * np.asarray(inv_freq, np.float32)[None, :]
emb = np.concatenate((freqs, freqs), axis=-1)
return (np.cos(emb) * scale).astype(np.float32), (np.sin(emb) * scale).astype(np.float32)
def conditioned_text(text, lang):
text = (text or "").strip()
return ("<|caption_begin|><|caption_end|><|accent_begin|><|accent_end|>"
"<|speaker_start|>None<|speaker_end|>"
f"<|lang_begin|>{'' if lang is None else lang}<|lang_end|>{text}")
class M5AX:
def __init__(self, model_dir, assets_dir, engine="ax", threads=4):
self.assets_dir = str(assets_dir)
self.cfg = json.loads((Path(assets_dir) / "config.json").read_text())
self.cap = self.cfg["cap"]
self.seq = self.cfg["seq"]
self.hop = self.cfg["hop_length"]
self.sr = self.cfg["sample_rate"]
self.noise_tiny = np.finfo(np.float32).tiny
engine_cls = AxEngine if engine == "ax" else OnnxEngine
suffix = ".axmodel" if engine == "ax" else ".onnx"
self.engines = {}
for name in ("lm_prefill", "lm_decode", "depth_prefill", "depth_step",
"codec_encoder", "codec_decoder"):
self.engines[name] = engine_cls(Path(model_dir) / f"{name}{suffix}")
a = Path(assets_dir)
self.text_emb = np.load(a / "text_embeddings.npy", mmap_mode="r")
self.audio_emb = np.load(a / "audio_embeddings.npy", mmap_mode="r")
self.head0 = np.load(a / "head0.npy")
self.audio_head = np.load(a / "audio_head.npy")
self.proj = np.load(a / "projection.npy")
self.binv = np.load(a / "backbone_inv_freq.npy")
self.dinv = np.load(a / "decoder_inv_freq.npy")
self.cinv = np.load(a / "codec_inv_freq.npy")
zq = np.load(a / "codec_enc_vq.npz")
self.evq_sem_w, self.evq_ac_w = zq["enc_sem_w"], zq["enc_ac_w"]
self.evq_sem_book, self.evq_ac_book = zq["enc_sem_book"], zq["enc_ac_book"]
z = np.load(a / "codec_rvq.npz")
self.book_sem = z["book_sem"][0]
self.book_rest = z["book_rest"]
self.ps_w, self.ps_b = z["proj_sem_w"], z["proj_sem_b"]
self.pr_w, self.pr_b = z["proj_rest_w"], z["proj_rest_b"]
meta = json.loads((a / "codec_meta.json").read_text())
self.clayers = meta["codec_layers"]
self.ckv = meta["codec_kv_heads"]
self.cdim = meta["codec_head_dim"]
self.cwindow = meta["codec_window"]
self.cstates = [s["shape"] for s in meta["codec_states"]]
self.cframes = meta["codec_frames"]
from tokenizers import Tokenizer
self.tokenizer = Tokenizer.from_file(str(a / "tokenizer.json"))
self.offsets = np.arange(16, dtype=np.int64) * self.cfg["audio_vocab"]
self.last_timing = {}
self.generated_codes = None
self._tables_ready = False
self._ref_cache = {}
self._prompt_cache = {}
prior_path = a / "codebook_prior.npy"
code_path = a / "mean_code.npy"
self.prior = np.load(prior_path) if prior_path.exists() else np.zeros((16, 512), np.float32)
self.mean_code = (np.load(code_path).astype(np.int64) if code_path.exists()
else np.arange(16, dtype=np.int64))
# precomputed RoPE tables (tokens are <= cap + 4)
top = self.cap + 8
cos_b, sin_b = rope_tables(self.binv, self.cfg["backbone_rope_scale"], np.arange(top))
self.bcos = cos_b.reshape(top, 64)
self.bsin = sin_b.reshape(top, 64)
cos_d, sin_d = rope_tables(self.dinv, self.cfg["decoder_rope_scale"], np.arange(self.cfg["depth_slots"] + 2))
self.dcos = cos_d.reshape(-1, 64)
self.dsin = sin_d.reshape(-1, 64)
ctop = 2048
cos_c, sin_c = rope_tables(self.cinv, 1.0, np.arange(ctop))
self.ccos = cos_c.reshape(ctop, 64)
self.csin = sin_c.reshape(ctop, 64)
# -------------------------------------------------------------- pieces
def _prepare_tables(self):
if self._tables_ready:
return
path = Path(self.assets_dir) / "projected_audio.npy"
if path.is_file():
self.proj_audio = np.load(path, mmap_mode="r")
else: # fallback: project every audio embedding row, fp32 [16*2051, 512]
self.proj_audio = np.asarray(self.audio_emb @ self.proj.T, np.float32)
self._tables_ready = True
def _run(self, name, feed):
return self.engines[name].run(feed)
@staticmethod
def _nearest(x, book):
x2 = (x * x).sum(1)[:, None]
b2 = (book * book).sum(1)[None, :]
d = x2 + b2 - 2.0 * (x @ book.T)
return np.argmin(d, axis=1)
def encode_reference(self, audio, key=None):
if key is not None and key in self._ref_cache:
return self._ref_cache[key].copy()
codes = self._encode_reference(audio)
if key is not None:
if len(self._ref_cache) > 4:
self._ref_cache.pop(next(iter(self._ref_cache)))
self._ref_cache[key] = codes
return codes
def _encode_reference(self, audio):
padded = np.zeros(self.cfg["codec_samples"], np.float32)
audio = audio[:self.cfg["codec_samples"]]
padded[:len(audio)] = audio
feat = self._run("codec_encoder", {"audio": padded[None]})[0] # [1,512,60]
x = feat[0].T
res = x @ self.evq_sem_w.T
book = self.evq_sem_book[0]
idx = self._nearest(res, book)
codes = [idx]
res = res - book[idx]
res = x @ self.evq_ac_w.T
for book in self.evq_ac_book[:15]:
idx = self._nearest(res, book)
codes.append(idx)
res = res - book[idx]
codes = np.stack(codes)
frames = max(1, int(math.ceil(len(audio) / self.hop)))
return codes[:, :frames].T.copy()
def codec_latent(self, codes, keep=16):
codes = np.asarray(codes, np.int64)
sem = self.book_sem[codes[:, 0]] @ self.ps_w.T + self.ps_b
acc = self.book_rest[0][codes[:, 1]].copy()
for i in range(1, max(0, keep - 1)):
acc += self.book_rest[i][codes[:, i + 1]]
rest = acc @ self.pr_w.T + self.pr_b
if keep < 16:
rest = rest + self.prior[keep:].sum(axis=0) # average residual of dropped codebooks
return np.ascontiguousarray((sem + rest).T, np.float32)
def _sample(self, logits, noise, allow_eos, topk=10, temperature=0.9, greedy=False):
if greedy:
idx = np.argpartition(logits, -topk)[-topk:]
best = idx[np.argmax(logits[idx])]
if not allow_eos and best == 0:
logits = logits.copy()
logits[0] = -1e5
idx = np.argpartition(logits, -topk)[-topk:]
best = idx[np.argmax(logits[idx])]
return int(best)
idx = np.argpartition(logits, -topk)[-topk:]
vals = logits[idx] / temperature
if not allow_eos and 0 in idx:
vals[idx == 0] = -1e5
vals = vals - vals.max()
np.exp(vals, out=vals)
vals /= vals.sum()
pick = int(np.argmax(vals / noise[idx]))
return int(idx[pick])
def _head(self, h, level):
return np.dot(h, self.audio_head[level])
def _local_rope(self, pos):
return self.bcos[pos].reshape(1, 1, 1, 64), self.bsin[pos].reshape(1, 1, 1, 64)
def _depth_rope(self, pos):
return self.dcos[pos].reshape(1, 1, 1, 64), self.dsin[pos].reshape(1, 1, 1, 64)
def _codec_mask(self, offset, n):
w = self.cwindow
key_abs = np.arange(w - 1 + n) + offset - (w - 1)
pos_abs = offset + np.arange(n)
allowed = ((key_abs[None, :] <= pos_abs[:, None]) & (key_abs[None, :] >= 0) &
(key_abs[None, :] >= pos_abs[:, None] - (w - 1)))
return np.where(allowed, 0.0, NEG).astype(np.float32)[None, None]
# ----------------------------------------------------------- generation
def warmup(self):
t = self.last_timing
try:
self.encode_reference(np.zeros(2400, np.float32))
self._run("lm_prefill", {
"hidden": np.zeros((1, self.seq, self.cfg["hidden"]), np.float32),
"cos": np.zeros((1, 1, self.seq, 64), np.float32),
"sin": np.zeros((1, 1, self.seq, 64), np.float32)})
self._run("lm_decode", {
"hidden": np.zeros((1, 1, self.cfg["hidden"]), np.float32),
"cos": np.zeros((1, 1, 1, 64), np.float32),
"sin": np.zeros((1, 1, 1, 64), np.float32),
"mask": np.zeros((1, 1, 1, self.cap + 1), np.float32),
"past": np.zeros((self.cfg["backbone_layers"], 2, 1,
self.cfg["backbone_kv_heads"], self.cap, 64), np.float32)})
self._run("depth_prefill", {"local": np.zeros((1, 2, self.cfg["hidden"]), np.float32)})
self._run("depth_step", {
"local": np.zeros((1, 1, self.cfg["hidden"]), np.float32),
"cos": np.zeros((1, 1, 1, 64), np.float32),
"sin": np.zeros((1, 1, 1, 64), np.float32),
"mask": np.zeros((1, 1, 1, self.cfg["depth_slots"] + 1), np.float32),
"past": np.zeros((self.cfg["decoder_layers"], 2, 1,
self.cfg["decoder_kv_heads"], self.cfg["depth_slots"], 64), np.float32)})
self.decode_codes(np.zeros((self.cframes, 16), np.int64))
finally:
t.clear()
# ------------------------------------------------------------ codec loop
class _CodecStream:
"""Incremental codec decoder state (KV window + 21 conv states)."""
def __init__(self, tts, keep=16):
self.tts = tts
self.keep = keep
self.past = np.zeros((tts.clayers, 2, 1, tts.ckv, tts.cwindow - 1, tts.cdim), np.float32)
self.states = [np.zeros(shape, np.float32) for shape in tts.cstates]
self.offset = 0
self.pending = np.zeros((0, 16), np.int64)
def snapshot(self):
return (self.past.copy(), [state.copy() for state in self.states],
self.offset, self.pending.copy())
def clone(self):
other = self.tts._CodecStream.__new__(self.tts._CodecStream)
other.tts = self.tts
other.past, states, other.offset, other.pending = self.snapshot()
other.states = states
return other
def push(self, codes, final=False):
self.pending = np.concatenate((self.pending, np.asarray(codes, np.int64)))
frames = self.tts.cframes
pcm = []
while len(self.pending) >= frames or (final and len(self.pending)):
block = self.pending[:frames]
self.pending = self.pending[len(block):]
pcm.append(self._chunk(block))
return np.concatenate(pcm) if pcm else np.zeros(0, np.float32)
def _chunk(self, codes):
tts = self.tts
n = len(codes)
latent = tts.codec_latent(codes, keep=self.keep)[None]
chunk = np.zeros((1, 512, tts.cframes), np.float32)
chunk[:, :, :n] = latent
off = self.offset
feed = {"latent": chunk, "past": self.past,
"cos": tts.ccos[off:off + tts.cframes][None, None],
"sin": tts.csin[off:off + tts.cframes][None, None],
"mask": tts._codec_mask(off, tts.cframes)}
for i, state in enumerate(self.states):
feed[f"conv_{i}"] = state
out = tts._run("codec_decoder", feed)
self.past = out[1]
self.states = out[2:]
self.offset += n
return out[0].reshape(-1)[:n * tts.hop]
def decode_codes(self, codes, keep=16):
stream = self._CodecStream(self, keep=keep)
return stream.push(codes, final=True)
# --------------------------------------------------------------- generation
def _prepare(self, text, prompt_wav, prompt_text, lang, seed, max_frames, prompt_codes=None, codebooks=16):
timing = {}
t0 = time.perf_counter()
key = None
stream = None
prefix = None
cached = False
if prompt_codes is None:
if isinstance(prompt_wav, (str, Path)):
stat = Path(prompt_wav).stat()
key = (str(Path(prompt_wav).resolve()), stat.st_size, stat.st_mtime_ns, codebooks)
entry = self._prompt_cache.get(key) if key is not None else None
if entry is not None:
prompt = entry["codes"].copy()
prefix = entry["prefix"].copy()
stream = entry["stream"].clone()
cached = True
else:
audio = read_wav(prompt_wav, self.sr)
prompt = self.encode_reference(audio, key=key)
else:
prompt = np.asarray(prompt_codes, np.int64)
timing["codec_enc"] = time.perf_counter() - t0
timing["ref_cached"] = cached
sentence = conditioned_text(prompt_text.strip() + text.strip(), lang)
ids = np.array(self.tokenizer.encode(sentence).ids, np.int64)
if prefix is None:
prefix = self.audio_emb[prompt + 1 + self.offsets].sum(axis=1).astype(np.float32)
hidden = np.concatenate((self.text_emb[ids], prefix), axis=0)
real = hidden.shape[0]
if real >= self.seq:
raise ValueError(f"prompt is {real} tokens, limit is {self.seq}")
hidden_pad = np.zeros((self.seq, self.cfg["hidden"]), np.float32)
hidden_pad[:real] = hidden
hidden_pad[real:] = hidden[-1]
cos_p = self.bcos[:self.seq].reshape(1, 1, self.seq, 64)
sin_p = self.bsin[:self.seq].reshape(1, 1, self.seq, 64)
t1 = time.perf_counter()
kv_out, kv = self._run("lm_prefill", {"hidden": hidden_pad[None],
"cos": cos_p, "sin": sin_p})
timing["prefill"] = time.perf_counter() - t1
last = kv_out[0, real - 1]
kv_cache = np.zeros((self.cfg["backbone_layers"], 2, 1,
self.cfg["backbone_kv_heads"], self.cap, 64), np.float32)
kv_cache[:, :, :, :, :real] = kv[:, :, :, :, :real]
return dict(prompt=prompt, real=real, last=last, kv_cache=kv_cache, pos=real,
timing=timing, t0=t0, prefix=prefix, stream=stream, key=key,
max_frames=min(max_frames, self.cap - real))
def synthesize_stream(self, text, prompt_wav, prompt_text, lang="Chinese", seed=1234,
max_frames=400, prompt_codes=None, first_frames=4, chunk_frames=8,
overlap=True, greedy=False, codebooks=16):
"""Yield 24 kHz PCM while the autoregressive LM is still running.
A background thread feeds the codec decoder so its ~6.6 ms/frame cost
overlaps with the next LM frame; the first audio packet is emitted after
first_frames (default 4 = 320 ms) of generated codes.
"""
import queue
import threading
self._prepare_tables()
state = self._prepare(text, prompt_wav, prompt_text, lang, seed, max_frames, prompt_codes, codebooks)
t0, timing = state["t0"], state["timing"]
prompt, real, pos = state["prompt"], state["real"], state["pos"]
kv_cache, last = state["kv_cache"], state["last"]
max_frames = state["max_frames"]
rng = np.random.default_rng(seed)
stream = state["stream"]
warm = stream is not None
if stream is None:
stream = self._CodecStream(self, keep=codebooks)
else:
stream.keep = codebooks
cache_key, prefix = state["key"], state["prefix"]
skip = [len(stream.pending) * self.hop if warm else len(prompt) * self.hop]
def decode(codes, final=False):
pcm = stream.push(codes, final)
if skip[0]:
k = min(skip[0], len(pcm))
pcm, skip[0] = pcm[k:], skip[0] - k
return pcm
jobs, pcm_q, errors = queue.Queue(), queue.Queue(), []
def codec_worker():
while True:
item = jobs.get()
if item is None:
break
codes, final, is_prompt = item
try:
pcm = decode(codes, final)
if is_prompt and cache_key is not None:
self._prompt_cache[cache_key] = dict(
codes=prompt.copy(), prefix=prefix.copy(), stream=stream.clone())
except Exception as error: # noqa: BLE001 - surface to caller
errors.append(error)
break
pcm_q.put(pcm)
if final:
break
pcm_q.put(None)
worker = threading.Thread(target=codec_worker, daemon=True) if overlap else None
if worker is not None:
worker.start()
if not warm:
jobs.put((prompt, False, True)) # prompt context decoded in the background
elif not warm:
decode(prompt) # sequential warmup of the prompt context
def collect(wait_for_audio):
out = []
while True:
try:
item = pcm_q.get(timeout=30) if wait_for_audio else pcm_q.get_nowait()
except queue.Empty:
break
if item is None:
break
if len(item):
out.append(item)
wait_for_audio = False # first packet found: drain the rest
if errors:
raise errors[0]
return out
depth_buf = np.zeros((self.cfg["decoder_layers"], 2, 1,
self.cfg["decoder_kv_heads"], self.cfg["depth_slots"], 64), np.float32)
depth_mask = np.full((1, 1, 1, self.cfg["depth_slots"] + 1), NEG, np.float32)
depth_feed = {"local": None, "mask": depth_mask, "cos": None, "sin": None, "past": depth_buf}
gen, eos = [], False
align = (-len(prompt)) % self.cframes
first_size = first_frames + align
threshold, queued = first_size, 0
ttft, ttfa, first_audio = None, None, True
ar_time = 0.0
chunks = 0
completed = False
try:
frame = 0
while frame < max_frames:
ta = time.perf_counter()
noise = rng.exponential(size=(16, self.cfg["audio_vocab"])).astype(np.float32)
np.maximum(noise, self.noise_tiny, out=noise)
c0 = self._sample(np.dot(self.head0, last), noise[0], frame > self.cfg["min_frames"], greedy=greedy)
local = np.empty((1, 2, self.cfg["hidden"]), np.float32)
np.dot(last, self.proj.T, out=local[0, 0])
np.dot(self.audio_emb[c0], self.proj.T, out=local[0, 1])
dh, dkv = self._run("depth_prefill", {"local": local})
depth_buf[:, :, :, :, :2] = dkv
h = dh[0]
codes = [c0]
for level in range(codebooks - 1):
ci = self._sample(np.dot(h, self.audio_head[level]), noise[level + 1], True, greedy=greedy)
codes.append(ci)
if level == codebooks - 2:
break
token_pos = level + 2
depth_feed["local"] = self.proj_audio[ci + (level + 1) * self.cfg["audio_vocab"]][None, None]
depth_mask[...] = NEG
depth_mask[..., :token_pos] = 0.0
depth_mask[..., -1] = 0.0
depth_feed["cos"] = self.dcos[token_pos].reshape(1, 1, 1, 64)
depth_feed["sin"] = self.dsin[token_pos].reshape(1, 1, 1, 64)
h_out, newkv = self._run("depth_step", depth_feed)
h = h_out[0]
depth_buf[:, :, :, :, token_pos] = newkv[:, :, :, :, 0]
codes = np.array(codes, np.int64)
frame += 1
if ttft is None:
ttft = time.perf_counter() - t0
if np.all(codes == 0):
eos = True
ar_time += time.perf_counter() - ta
break
gen.append(np.clip(codes - 1, 0, self.cfg["audio_vocab"] - 2))
if codebooks < 16:
# dropped codebooks contribute their average residual vector
next_hidden = (self.audio_emb[codes + self.offsets[:codebooks]].sum(axis=0)
+ self.prior[codebooks:].sum(axis=0))[None]
else:
next_hidden = self.audio_emb[codes + self.offsets].sum(axis=0)[None]
bmask = np.full((1, 1, 1, self.cap + 1), NEG, np.float32)
bmask[..., :pos] = 0.0
bmask[..., -1] = 0.0
out, newkv = self._run("lm_decode", {"hidden": next_hidden[None], "mask": bmask,
"cos": self.bcos[pos].reshape(1, 1, 1, 64),
"sin": self.bsin[pos].reshape(1, 1, 1, 64),
"past": kv_cache})
kv_cache[:, :, :, :, pos] = newkv[:, :, :, :, 0]
last = out[0]
pos += 1
ar_time += time.perf_counter() - ta
if len(gen) - queued >= threshold:
block = np.stack(gen[queued:])
queued = len(gen)
threshold = chunk_frames
if worker is not None:
jobs.put((block, False, False))
pcm = collect(first_audio)
else:
pcm = [decode(block)]
if pcm:
if ttfa is None:
ttfa = time.perf_counter() - t0
first_audio = False
chunks += 1
yield np.concatenate(pcm)
completed = True
finally:
if not completed:
jobs.put(None)
if len(gen) > queued:
if worker is not None:
jobs.put((np.stack(gen[queued:]), False, False))
else:
pcm = decode(np.stack(gen[queued:]))
if len(pcm):
if ttfa is None:
ttfa = time.perf_counter() - t0
chunks += 1
yield pcm
if worker is not None:
jobs.put((np.zeros((0, 16), np.int64), True, False))
worker.join()
tail = collect(False)
else:
tail = [decode(np.zeros((0, 16), np.int64), final=True)]
if tail:
if ttfa is None:
ttfa = time.perf_counter() - t0
chunks += 1
yield np.concatenate(tail)
total = time.perf_counter() - t0
seconds = len(gen) * self.hop / self.sr
self.generated_codes = np.stack(gen) if gen else np.zeros((0, 16), np.int64)
timing.update(ar_loop=ar_time, codec_dec=max(0.0, total - ar_time - timing["codec_enc"] -
timing["prefill"]),
frames=len(gen), eos=eos, prompt_frames=len(prompt), tokens=real,
audio_seconds=seconds, total=total,
rtf=total / max(seconds, 1e-6), seed=seed, text=text, lang=lang,
ttft=ttft, ttfa=ttfa, chunks=chunks, streaming=True,
codebooks=codebooks, max_frames=max_frames)
self.last_timing = timing
def synthesize(self, text, prompt_wav, prompt_text, **kwargs):
kwargs.pop("stream", None)
chunks = list(self.synthesize_stream(text, prompt_wav, prompt_text, **kwargs))
if not chunks:
return np.zeros(0, np.float32)
wav = np.concatenate(chunks)
self.last_timing["streaming"] = False
return wav
def generate(self, text, prompt_wav, prompt_text, lang="Chinese", seed=1234,
max_frames=400, stream=False, **kwargs):
"""Single call: returns PCM (or a chunk generator when stream=True)."""
if stream:
return self.synthesize_stream(text, prompt_wav, prompt_text, lang=lang, seed=seed,
max_frames=max_frames, **kwargs)
return self.synthesize(text, prompt_wav, prompt_text, lang=lang, seed=seed,
max_frames=max_frames, **kwargs)