Instructions to use moonbox-dev/VoxCPM2-ONNX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- VoxCPM
How to use moonbox-dev/VoxCPM2-ONNX with VoxCPM:
import soundfile as sf from voxcpm import VoxCPM model = VoxCPM.from_pretrained("moonbox-dev/VoxCPM2-ONNX") wav = model.generate( text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.", prompt_wav_path=None, # optional: path to a prompt speech for voice cloning prompt_text=None, # optional: reference text cfg_value=2.0, # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse inference_timesteps=10, # LocDiT inference timesteps, higher for better result, lower for fast speed normalize=True, # enable external TN tool denoise=True, # enable external Denoise tool retry_badcase=True, # enable retrying mode for some bad cases (unstoppable) retry_badcase_max_times=3, # maximum retrying times retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech ) sf.write("output.wav", wav, 16000) print("saved: output.wav") - Notebooks
- Google Colab
- Kaggle
int8 build of VoxCPM2
Browse files- int8/base_lm.onnx +3 -0
- int8/estimator.onnx +3 -0
- int8/feat_encoder.onnx +3 -0
- int8/fsq.onnx +3 -0
- int8/fusion.onnx +3 -0
- int8/heads.onnx +3 -0
- int8/multichar_tokens.json +0 -0
- int8/pipeline.py +174 -0
- int8/residual_lm.onnx +3 -0
- int8/text_embed.onnx +3 -0
- int8/tokenizer.json +0 -0
- int8/vae_decoder.onnx +3 -0
- int8/vae_encoder.onnx +3 -0
int8/base_lm.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e498cb38af6dc56a49f50b038e03e0597387cdbc00088b4a39478ec505bca3bb
|
| 3 |
+
size 1428709763
|
int8/estimator.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:890a11ed35615cc6d380ba42479c6123dacff119a8d20bd186366416022c68a5
|
| 3 |
+
size 213546461
|
int8/feat_encoder.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0d3b35cac7f34a1eda687d7c7517195352f82d4643414891d28b3f4eb4c2d0c0
|
| 3 |
+
size 244701917
|
int8/fsq.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e53a24ebbede24107bb4b4a1f2462bf04f1ea1bf0cf9bfbaa27b5f0d377dcaf
|
| 3 |
+
size 2121609
|
int8/fusion.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2ce552d569a29bd96047c28dd3ab4506a7b06ed8f9a68ac7c33618bc54270bfa
|
| 3 |
+
size 8407649
|
int8/heads.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9a60167d9bacd73aa8d207a097960977f788398f6f4abff7a3a9020a47061964
|
| 3 |
+
size 8441542
|
int8/multichar_tokens.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
int8/pipeline.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""VoxCPM2 voice cloning with ONNX Runtime: the graphs in this folder plus numpy for the rest, following voxcpm 2.0.3's
|
| 2 |
+
`VoxCPM.generate(text, prompt_wav_path, prompt_text, reference_wav_path)` with its defaults (10 Euler steps of the
|
| 3 |
+
local DiT, classifier-free guidance 2.0 with CFG-zero*, at most 6 x the text's tokens + 10 patches, no retry).
|
| 4 |
+
48 kHz output. Needs numpy, onnxruntime (or onnxruntime-gpu), tokenizers, soundfile and soxr.
|
| 5 |
+
|
| 6 |
+
python fp16/pipeline.py "Text to say." out.wav --ref voice.wav --ref-text "What voice.wav says." [--cuda] [--seed 0]
|
| 7 |
+
|
| 8 |
+
The reference audio is used twice, as the original's documented cloning call does: as the reference prefix and as
|
| 9 |
+
the prompt the generation continues from.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
SR_IN, SR_OUT = 16000, 48000
|
| 20 |
+
PATCH, CHUNK, DEC_CHUNK = 4, 640, 1920 # latent frames per patch; samples per latent frame in (16 kHz) / out (48 kHz)
|
| 21 |
+
AUDIO_START, REF_START, REF_END = 101, 103, 104
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class VoxCPM2:
|
| 25 |
+
def __init__(self, folder, provider="cpu", threads=4):
|
| 26 |
+
import onnxruntime as ort
|
| 27 |
+
from tokenizers import Tokenizer
|
| 28 |
+
folder = Path(folder)
|
| 29 |
+
self.tok = Tokenizer.from_file(str(folder / "tokenizer.json"))
|
| 30 |
+
self.multichar = set(json.loads((folder / "multichar_tokens.json").read_text(encoding="utf-8")))
|
| 31 |
+
if provider != "cpu" and hasattr(ort, "preload_dlls"):
|
| 32 |
+
try:
|
| 33 |
+
ort.preload_dlls()
|
| 34 |
+
except Exception:
|
| 35 |
+
pass
|
| 36 |
+
so = ort.SessionOptions()
|
| 37 |
+
so.intra_op_num_threads = threads
|
| 38 |
+
prov = ["CPUExecutionProvider"] if provider == "cpu" else [
|
| 39 |
+
("CUDAExecutionProvider", {"arena_extend_strategy": "kSameAsRequested"}), "CPUExecutionProvider"]
|
| 40 |
+
names = ["vae_encoder", "vae_decoder", "feat_encoder", "text_embed", "base_lm", "residual_lm", "fsq",
|
| 41 |
+
"fusion", "heads", "estimator"]
|
| 42 |
+
self.s = {n: ort.InferenceSession(str(folder / f"{n}.onnx"), so, providers=prov) for n in names}
|
| 43 |
+
self.ort = ort
|
| 44 |
+
self.cuda = "CUDAExecutionProvider" in self.s["base_lm"].get_providers()
|
| 45 |
+
if provider != "cpu" and not self.cuda:
|
| 46 |
+
print("note: CUDA provider not available, running on the CPU", flush=True)
|
| 47 |
+
self.ro = ort.RunOptions()
|
| 48 |
+
if self.cuda:
|
| 49 |
+
self.ro.add_run_config_entry("memory.enable_memory_arena_shrinkage", "gpu:0")
|
| 50 |
+
self.lm = {}
|
| 51 |
+
for n in ("base_lm", "residual_lm"):
|
| 52 |
+
ins = {i.name: i for i in self.s[n].get_inputs()}
|
| 53 |
+
pk = ins["past_key"]
|
| 54 |
+
self.lm[n] = (np.float16 if "float16" in pk.type else np.float32, pk.shape[0], pk.shape[2], pk.shape[4],
|
| 55 |
+
np.float16 if "float16" in ins["inputs_embeds"].type else np.float32)
|
| 56 |
+
|
| 57 |
+
def run(self, name, **feeds):
|
| 58 |
+
return self.s[name].run(None, feeds, self.ro)
|
| 59 |
+
|
| 60 |
+
# -- text -----------------------------------------------------------------------------------------------------
|
| 61 |
+
def ids(self, text):
|
| 62 |
+
"""LlamaTokenizerFast with multi-character Chinese tokens split into characters (mask_multichar_chinese_tokens)."""
|
| 63 |
+
out = []
|
| 64 |
+
for t in self.tok.encode(text, add_special_tokens=False).tokens:
|
| 65 |
+
c = t.replace("▁", "")
|
| 66 |
+
if c in self.multichar:
|
| 67 |
+
out += [self.tok.token_to_id(ch) for ch in c]
|
| 68 |
+
else:
|
| 69 |
+
out.append(self.tok.token_to_id(t))
|
| 70 |
+
return out
|
| 71 |
+
|
| 72 |
+
# -- audio ----------------------------------------------------------------------------------------------------
|
| 73 |
+
def encode_wav(self, wav16, left=False):
|
| 74 |
+
n = PATCH * CHUNK
|
| 75 |
+
pad = (-len(wav16)) % n
|
| 76 |
+
w = np.pad(wav16, (pad, 0) if left else (0, pad)).astype(np.float32)
|
| 77 |
+
mu = self.run("vae_encoder", wav=w[None, None])[0][0] # [64, T]
|
| 78 |
+
return mu.reshape(64, -1, PATCH).transpose(1, 2, 0) # [T / 4, 4, 64]
|
| 79 |
+
|
| 80 |
+
# -- language models ------------------------------------------------------------------------------------------
|
| 81 |
+
def lm_run(self, name, x, past):
|
| 82 |
+
dt, L, kvh, hd, edt = self.lm[name]
|
| 83 |
+
if past is None:
|
| 84 |
+
z = np.zeros((L, 1, kvh, 0, hd), dt)
|
| 85 |
+
past = (z, z)
|
| 86 |
+
h, k, v = self.run(name, inputs_embeds=x.astype(edt)[None], past_key=past[0], past_value=past[1])
|
| 87 |
+
return h[0].astype(np.float32), (k, v)
|
| 88 |
+
|
| 89 |
+
# -- local DiT: UnifiedCFM.forward / solve_euler with CFG-zero* -----------------------------------------------
|
| 90 |
+
def cfm(self, mu, cond, rng, n_steps=10, cfg=2.0):
|
| 91 |
+
x = rng.standard_normal((1, 64, PATCH)).astype(np.float32)
|
| 92 |
+
t_span = np.linspace(1, 0, n_steps + 1, dtype=np.float32)
|
| 93 |
+
t_span = t_span + (np.cos(np.pi / 2 * t_span) - 1 + t_span)
|
| 94 |
+
t, dt = t_span[0], t_span[0] - t_span[1]
|
| 95 |
+
zero_init = max(1, int(len(t_span) * 0.04))
|
| 96 |
+
mu_in = np.concatenate([mu, np.zeros_like(mu)], 0).astype(np.float32)
|
| 97 |
+
cond_in = np.concatenate([cond, cond], 0).astype(np.float32)
|
| 98 |
+
for step in range(1, len(t_span)):
|
| 99 |
+
if step <= zero_init:
|
| 100 |
+
d = np.zeros_like(x)
|
| 101 |
+
else:
|
| 102 |
+
v = self.run("estimator", x=np.concatenate([x, x], 0), mu=mu_in,
|
| 103 |
+
t=np.array([t, t], np.float32), cond=cond_in, dt=np.zeros(2, np.float32))[0]
|
| 104 |
+
pos, neg = v[:1], v[1:]
|
| 105 |
+
st = (pos * neg).sum() / ((neg ** 2).sum() + 1e-8)
|
| 106 |
+
d = neg * st + cfg * (pos - neg * st)
|
| 107 |
+
x = (x - dt * d).astype(np.float32)
|
| 108 |
+
t = t - dt
|
| 109 |
+
if step < len(t_span) - 1:
|
| 110 |
+
dt = t - t_span[step + 1]
|
| 111 |
+
return x # [1, 64, 4]
|
| 112 |
+
|
| 113 |
+
def __call__(self, text, ref_wav, ref_sr, ref_text, seed=0, min_len=2, max_len=2000):
|
| 114 |
+
import re
|
| 115 |
+
import soxr
|
| 116 |
+
rng = np.random.default_rng(seed)
|
| 117 |
+
text = re.sub(r"\s+", " ", text.replace("\n", " ")) # as VoxCPM.generate cleans it
|
| 118 |
+
w = np.asarray(ref_wav, np.float32)
|
| 119 |
+
w = w.mean(-1) if w.ndim > 1 else w
|
| 120 |
+
w16 = soxr.resample(w, ref_sr, SR_IN, quality="HQ").astype(np.float32) if ref_sr != SR_IN else w
|
| 121 |
+
ref_feat, prompt_feat = self.encode_wav(w16), self.encode_wav(w16, left=True)
|
| 122 |
+
text_tok = self.ids(ref_text + text) + [AUDIO_START]
|
| 123 |
+
R, P, T = len(ref_feat), len(prompt_feat), len(text_tok)
|
| 124 |
+
z1 = np.zeros((1, PATCH, 64), np.float32)
|
| 125 |
+
tokens = np.array([REF_START] + [0] * R + [REF_END] + text_tok + [0] * P, np.int64)
|
| 126 |
+
feats = np.concatenate([z1, ref_feat, z1, np.zeros((T, PATCH, 64), np.float32), prompt_feat], 0)
|
| 127 |
+
tmask = np.array([1] + [0] * R + [1] + [1] * T + [0] * P, np.float32)[:, None]
|
| 128 |
+
amask = 1 - tmask
|
| 129 |
+
max_len = min(int(len(self.ids(text)) * 6.0 + 10), max_len)
|
| 130 |
+
|
| 131 |
+
feat_embed = self.run("feat_encoder", patches=feats[None])[0][0]
|
| 132 |
+
text_embed = self.run("text_embed", input_ids=tokens[None])[0][0]
|
| 133 |
+
combined = tmask * text_embed + amask * feat_embed
|
| 134 |
+
context = min(PATCH - 1, P) # streaming_prefix_len - 1 continuation patches
|
| 135 |
+
seq = [feats[-context:]] if context else []
|
| 136 |
+
cond = feats[-1] # [4, 64]
|
| 137 |
+
enc, kv_b = self.lm_run("base_lm", combined, None)
|
| 138 |
+
enc = self.run("fsq", hidden=enc[None])[0][0] * amask + enc * tmask
|
| 139 |
+
lm_hidden = enc[-1]
|
| 140 |
+
res_in = self.run("fusion", lm_out=enc[None], feat_embed=(amask * feat_embed)[None])[0][0]
|
| 141 |
+
res, kv_r = self.lm_run("residual_lm", res_in, None)
|
| 142 |
+
res_hidden = res[-1]
|
| 143 |
+
for i in range(max_len):
|
| 144 |
+
dit, stop = self.run("heads", lm_hidden=lm_hidden[None], residual_hidden=res_hidden[None])
|
| 145 |
+
pred = self.cfm(dit, cond.T[None], rng)[0].T # [4, 64]
|
| 146 |
+
curr = self.run("feat_encoder", patches=pred[None, None])[0][0, 0]
|
| 147 |
+
seq.append(pred[None])
|
| 148 |
+
cond = pred
|
| 149 |
+
if i > min_len and int(stop[0].argmax()) == 1:
|
| 150 |
+
break
|
| 151 |
+
h, kv_b = self.lm_run("base_lm", curr[None], kv_b)
|
| 152 |
+
lm_hidden = self.run("fsq", hidden=h[None])[0][0, 0]
|
| 153 |
+
r_in = self.run("fusion", lm_out=lm_hidden[None, None], feat_embed=curr[None, None])[0][0]
|
| 154 |
+
r, kv_r = self.lm_run("residual_lm", r_in, kv_r)
|
| 155 |
+
res_hidden = r[0]
|
| 156 |
+
lat = np.concatenate(seq, 0) # [t, 4, 64]
|
| 157 |
+
lat = lat.reshape(-1, 64).T[None].astype(np.float32) # [1, 64, t * 4]
|
| 158 |
+
wav = self.run("vae_decoder", latent=lat)[0][0, 0]
|
| 159 |
+
return wav[context * PATCH * DEC_CHUNK:].astype(np.float32), SR_OUT
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
import argparse
|
| 164 |
+
import soundfile as sf
|
| 165 |
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
| 166 |
+
ap.add_argument("text"); ap.add_argument("output")
|
| 167 |
+
ap.add_argument("--ref", required=True); ap.add_argument("--ref-text", required=True)
|
| 168 |
+
ap.add_argument("--dir", default=str(Path(__file__).resolve().parent))
|
| 169 |
+
ap.add_argument("--cuda", action="store_true"); ap.add_argument("--seed", type=int, default=0)
|
| 170 |
+
a = ap.parse_args()
|
| 171 |
+
w, sr = sf.read(a.ref, dtype="float32", always_2d=True)
|
| 172 |
+
tts = VoxCPM2(a.dir, "cuda" if a.cuda else "cpu")
|
| 173 |
+
y, osr = tts(a.text, w.mean(1), sr, a.ref_text, a.seed)
|
| 174 |
+
sf.write(a.output, y, osr)
|
int8/residual_lm.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e557865390b838ab14cfd9651fd66da1a2db708c7605b864637b5347f382b853
|
| 3 |
+
size 403368176
|
int8/text_embed.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:06477af2a9f1a4145a76bacd9166351007498774aca795bde196dbaa97de7158
|
| 3 |
+
size 150715656
|
int8/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
int8/vae_decoder.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:126eba421426b73994c9847322fb8e4a5fd48432e4dcdd532bf5a621dfe2334c
|
| 3 |
+
size 46309386
|
int8/vae_encoder.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b5b89a6b44be1a114af5688de15f07c777158bb14611c7e45c7d1ead80760dd8
|
| 3 |
+
size 48384149
|