jennyxue commited on
Commit
009457f
·
verified ·
1 Parent(s): 92f2d04

int8 build of VoxCPM2

Browse files
int8/base_lm.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e498cb38af6dc56a49f50b038e03e0597387cdbc00088b4a39478ec505bca3bb
3
+ size 1428709763
int8/estimator.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:890a11ed35615cc6d380ba42479c6123dacff119a8d20bd186366416022c68a5
3
+ size 213546461
int8/feat_encoder.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d3b35cac7f34a1eda687d7c7517195352f82d4643414891d28b3f4eb4c2d0c0
3
+ size 244701917
int8/fsq.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e53a24ebbede24107bb4b4a1f2462bf04f1ea1bf0cf9bfbaa27b5f0d377dcaf
3
+ size 2121609
int8/fusion.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ce552d569a29bd96047c28dd3ab4506a7b06ed8f9a68ac7c33618bc54270bfa
3
+ size 8407649
int8/heads.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a60167d9bacd73aa8d207a097960977f788398f6f4abff7a3a9020a47061964
3
+ size 8441542
int8/multichar_tokens.json ADDED
The diff for this file is too large to render. See raw diff
 
int8/pipeline.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """VoxCPM2 voice cloning with ONNX Runtime: the graphs in this folder plus numpy for the rest, following voxcpm 2.0.3's
2
+ `VoxCPM.generate(text, prompt_wav_path, prompt_text, reference_wav_path)` with its defaults (10 Euler steps of the
3
+ local DiT, classifier-free guidance 2.0 with CFG-zero*, at most 6 x the text's tokens + 10 patches, no retry).
4
+ 48 kHz output. Needs numpy, onnxruntime (or onnxruntime-gpu), tokenizers, soundfile and soxr.
5
+
6
+ python fp16/pipeline.py "Text to say." out.wav --ref voice.wav --ref-text "What voice.wav says." [--cuda] [--seed 0]
7
+
8
+ The reference audio is used twice, as the original's documented cloning call does: as the reference prefix and as
9
+ the prompt the generation continues from.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ from pathlib import Path
16
+
17
+ import numpy as np
18
+
19
+ SR_IN, SR_OUT = 16000, 48000
20
+ PATCH, CHUNK, DEC_CHUNK = 4, 640, 1920 # latent frames per patch; samples per latent frame in (16 kHz) / out (48 kHz)
21
+ AUDIO_START, REF_START, REF_END = 101, 103, 104
22
+
23
+
24
+ class VoxCPM2:
25
+ def __init__(self, folder, provider="cpu", threads=4):
26
+ import onnxruntime as ort
27
+ from tokenizers import Tokenizer
28
+ folder = Path(folder)
29
+ self.tok = Tokenizer.from_file(str(folder / "tokenizer.json"))
30
+ self.multichar = set(json.loads((folder / "multichar_tokens.json").read_text(encoding="utf-8")))
31
+ if provider != "cpu" and hasattr(ort, "preload_dlls"):
32
+ try:
33
+ ort.preload_dlls()
34
+ except Exception:
35
+ pass
36
+ so = ort.SessionOptions()
37
+ so.intra_op_num_threads = threads
38
+ prov = ["CPUExecutionProvider"] if provider == "cpu" else [
39
+ ("CUDAExecutionProvider", {"arena_extend_strategy": "kSameAsRequested"}), "CPUExecutionProvider"]
40
+ names = ["vae_encoder", "vae_decoder", "feat_encoder", "text_embed", "base_lm", "residual_lm", "fsq",
41
+ "fusion", "heads", "estimator"]
42
+ self.s = {n: ort.InferenceSession(str(folder / f"{n}.onnx"), so, providers=prov) for n in names}
43
+ self.ort = ort
44
+ self.cuda = "CUDAExecutionProvider" in self.s["base_lm"].get_providers()
45
+ if provider != "cpu" and not self.cuda:
46
+ print("note: CUDA provider not available, running on the CPU", flush=True)
47
+ self.ro = ort.RunOptions()
48
+ if self.cuda:
49
+ self.ro.add_run_config_entry("memory.enable_memory_arena_shrinkage", "gpu:0")
50
+ self.lm = {}
51
+ for n in ("base_lm", "residual_lm"):
52
+ ins = {i.name: i for i in self.s[n].get_inputs()}
53
+ pk = ins["past_key"]
54
+ self.lm[n] = (np.float16 if "float16" in pk.type else np.float32, pk.shape[0], pk.shape[2], pk.shape[4],
55
+ np.float16 if "float16" in ins["inputs_embeds"].type else np.float32)
56
+
57
+ def run(self, name, **feeds):
58
+ return self.s[name].run(None, feeds, self.ro)
59
+
60
+ # -- text -----------------------------------------------------------------------------------------------------
61
+ def ids(self, text):
62
+ """LlamaTokenizerFast with multi-character Chinese tokens split into characters (mask_multichar_chinese_tokens)."""
63
+ out = []
64
+ for t in self.tok.encode(text, add_special_tokens=False).tokens:
65
+ c = t.replace("▁", "")
66
+ if c in self.multichar:
67
+ out += [self.tok.token_to_id(ch) for ch in c]
68
+ else:
69
+ out.append(self.tok.token_to_id(t))
70
+ return out
71
+
72
+ # -- audio ----------------------------------------------------------------------------------------------------
73
+ def encode_wav(self, wav16, left=False):
74
+ n = PATCH * CHUNK
75
+ pad = (-len(wav16)) % n
76
+ w = np.pad(wav16, (pad, 0) if left else (0, pad)).astype(np.float32)
77
+ mu = self.run("vae_encoder", wav=w[None, None])[0][0] # [64, T]
78
+ return mu.reshape(64, -1, PATCH).transpose(1, 2, 0) # [T / 4, 4, 64]
79
+
80
+ # -- language models ------------------------------------------------------------------------------------------
81
+ def lm_run(self, name, x, past):
82
+ dt, L, kvh, hd, edt = self.lm[name]
83
+ if past is None:
84
+ z = np.zeros((L, 1, kvh, 0, hd), dt)
85
+ past = (z, z)
86
+ h, k, v = self.run(name, inputs_embeds=x.astype(edt)[None], past_key=past[0], past_value=past[1])
87
+ return h[0].astype(np.float32), (k, v)
88
+
89
+ # -- local DiT: UnifiedCFM.forward / solve_euler with CFG-zero* -----------------------------------------------
90
+ def cfm(self, mu, cond, rng, n_steps=10, cfg=2.0):
91
+ x = rng.standard_normal((1, 64, PATCH)).astype(np.float32)
92
+ t_span = np.linspace(1, 0, n_steps + 1, dtype=np.float32)
93
+ t_span = t_span + (np.cos(np.pi / 2 * t_span) - 1 + t_span)
94
+ t, dt = t_span[0], t_span[0] - t_span[1]
95
+ zero_init = max(1, int(len(t_span) * 0.04))
96
+ mu_in = np.concatenate([mu, np.zeros_like(mu)], 0).astype(np.float32)
97
+ cond_in = np.concatenate([cond, cond], 0).astype(np.float32)
98
+ for step in range(1, len(t_span)):
99
+ if step <= zero_init:
100
+ d = np.zeros_like(x)
101
+ else:
102
+ v = self.run("estimator", x=np.concatenate([x, x], 0), mu=mu_in,
103
+ t=np.array([t, t], np.float32), cond=cond_in, dt=np.zeros(2, np.float32))[0]
104
+ pos, neg = v[:1], v[1:]
105
+ st = (pos * neg).sum() / ((neg ** 2).sum() + 1e-8)
106
+ d = neg * st + cfg * (pos - neg * st)
107
+ x = (x - dt * d).astype(np.float32)
108
+ t = t - dt
109
+ if step < len(t_span) - 1:
110
+ dt = t - t_span[step + 1]
111
+ return x # [1, 64, 4]
112
+
113
+ def __call__(self, text, ref_wav, ref_sr, ref_text, seed=0, min_len=2, max_len=2000):
114
+ import re
115
+ import soxr
116
+ rng = np.random.default_rng(seed)
117
+ text = re.sub(r"\s+", " ", text.replace("\n", " ")) # as VoxCPM.generate cleans it
118
+ w = np.asarray(ref_wav, np.float32)
119
+ w = w.mean(-1) if w.ndim > 1 else w
120
+ w16 = soxr.resample(w, ref_sr, SR_IN, quality="HQ").astype(np.float32) if ref_sr != SR_IN else w
121
+ ref_feat, prompt_feat = self.encode_wav(w16), self.encode_wav(w16, left=True)
122
+ text_tok = self.ids(ref_text + text) + [AUDIO_START]
123
+ R, P, T = len(ref_feat), len(prompt_feat), len(text_tok)
124
+ z1 = np.zeros((1, PATCH, 64), np.float32)
125
+ tokens = np.array([REF_START] + [0] * R + [REF_END] + text_tok + [0] * P, np.int64)
126
+ feats = np.concatenate([z1, ref_feat, z1, np.zeros((T, PATCH, 64), np.float32), prompt_feat], 0)
127
+ tmask = np.array([1] + [0] * R + [1] + [1] * T + [0] * P, np.float32)[:, None]
128
+ amask = 1 - tmask
129
+ max_len = min(int(len(self.ids(text)) * 6.0 + 10), max_len)
130
+
131
+ feat_embed = self.run("feat_encoder", patches=feats[None])[0][0]
132
+ text_embed = self.run("text_embed", input_ids=tokens[None])[0][0]
133
+ combined = tmask * text_embed + amask * feat_embed
134
+ context = min(PATCH - 1, P) # streaming_prefix_len - 1 continuation patches
135
+ seq = [feats[-context:]] if context else []
136
+ cond = feats[-1] # [4, 64]
137
+ enc, kv_b = self.lm_run("base_lm", combined, None)
138
+ enc = self.run("fsq", hidden=enc[None])[0][0] * amask + enc * tmask
139
+ lm_hidden = enc[-1]
140
+ res_in = self.run("fusion", lm_out=enc[None], feat_embed=(amask * feat_embed)[None])[0][0]
141
+ res, kv_r = self.lm_run("residual_lm", res_in, None)
142
+ res_hidden = res[-1]
143
+ for i in range(max_len):
144
+ dit, stop = self.run("heads", lm_hidden=lm_hidden[None], residual_hidden=res_hidden[None])
145
+ pred = self.cfm(dit, cond.T[None], rng)[0].T # [4, 64]
146
+ curr = self.run("feat_encoder", patches=pred[None, None])[0][0, 0]
147
+ seq.append(pred[None])
148
+ cond = pred
149
+ if i > min_len and int(stop[0].argmax()) == 1:
150
+ break
151
+ h, kv_b = self.lm_run("base_lm", curr[None], kv_b)
152
+ lm_hidden = self.run("fsq", hidden=h[None])[0][0, 0]
153
+ r_in = self.run("fusion", lm_out=lm_hidden[None, None], feat_embed=curr[None, None])[0][0]
154
+ r, kv_r = self.lm_run("residual_lm", r_in, kv_r)
155
+ res_hidden = r[0]
156
+ lat = np.concatenate(seq, 0) # [t, 4, 64]
157
+ lat = lat.reshape(-1, 64).T[None].astype(np.float32) # [1, 64, t * 4]
158
+ wav = self.run("vae_decoder", latent=lat)[0][0, 0]
159
+ return wav[context * PATCH * DEC_CHUNK:].astype(np.float32), SR_OUT
160
+
161
+
162
+ if __name__ == "__main__":
163
+ import argparse
164
+ import soundfile as sf
165
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
166
+ ap.add_argument("text"); ap.add_argument("output")
167
+ ap.add_argument("--ref", required=True); ap.add_argument("--ref-text", required=True)
168
+ ap.add_argument("--dir", default=str(Path(__file__).resolve().parent))
169
+ ap.add_argument("--cuda", action="store_true"); ap.add_argument("--seed", type=int, default=0)
170
+ a = ap.parse_args()
171
+ w, sr = sf.read(a.ref, dtype="float32", always_2d=True)
172
+ tts = VoxCPM2(a.dir, "cuda" if a.cuda else "cpu")
173
+ y, osr = tts(a.text, w.mean(1), sr, a.ref_text, a.seed)
174
+ sf.write(a.output, y, osr)
int8/residual_lm.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e557865390b838ab14cfd9651fd66da1a2db708c7605b864637b5347f382b853
3
+ size 403368176
int8/text_embed.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06477af2a9f1a4145a76bacd9166351007498774aca795bde196dbaa97de7158
3
+ size 150715656
int8/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
int8/vae_decoder.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:126eba421426b73994c9847322fb8e4a5fd48432e4dcdd532bf5a621dfe2334c
3
+ size 46309386
int8/vae_encoder.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5b89a6b44be1a114af5688de15f07c777158bb14611c7e45c7d1ead80760dd8
3
+ size 48384149