File size: 3,253 Bytes
ca95b32 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | import os
REF_WAV = os.environ.get("REF_WAV", "reference.wav") # clean 5-10 s speech, 24 kHz or any rate
REF_TEXT = os.environ.get("REF_TEXT", "") # exact transcript of REF_WAV (gen_eval.py only)
import torch, os, time, numpy as np
import torch.nn.functional as F
torch.set_num_threads(4)
from transformers import HiggsAudioV2TokenizerModel
m=HiggsAudioV2TokenizerModel.from_pretrained("eustlb/higgs-audio-v2-tokenizer").eval()
out="onnx_fp16"; os.makedirs(out,exist_ok=True)
print("semantic_downsample_factor",m.config.semantic_downsample_factor,"targets",m.config.target_bandwidths,"nq",m.quantizer.num_quantizers)
class Dec(torch.nn.Module):
def __init__(s,m): super().__init__(); s.m=m
def forward(s,codes): # [1,8,T] int64 -> [1,1,T*960]
q=0
for i,qz in enumerate(s.m.quantizer.quantizers): q=q+qz.decode(codes[:,i])
return s.m.acoustic_decoder(s.m.fc2(q.transpose(1,2)).transpose(1,2))
class Enc(torch.nn.Module):
def __init__(s,m,pad_acoustic): super().__init__(); s.m=m; s.pad_acoustic=pad_acoustic
def forward(s,wav24,wav16): # [1,1,N], [1,N16] (16 kHz resample done by the caller)
m=s.m
x=F.pad(wav16,(160,160))
hs=m.semantic_model(x,output_hidden_states=True).hidden_states
sem=torch.stack(list(hs),dim=1).mean(dim=1)
if m.config.semantic_downsample_factor>1: sem=sem[:,::m.config.semantic_downsample_factor,:]
e_sem=m.encoder_semantic(sem.transpose(1,2))
e_ac=m.acoustic_encoder(F.pad(wav24,(m.pad,m.pad)) if s.pad_acoustic else wav24)
e=torch.cat([e_ac,e_sem],dim=1)
e=m.fc(e.transpose(1,2)).transpose(1,2)
return m.quantizer.encode(e,m.config.target_bandwidths[-1]).transpose(0,1)
import soundfile as sf, torchaudio
a,sr=sf.read(REF_WAV); a=torch.tensor(a,dtype=torch.float32)
a=torchaudio.functional.resample(a,sr,24000) if sr!=24000 else a
a=F.pad(a,(0,-a.numel()%960)); w24=a[None,None]
w16=torchaudio.functional.resample(w24,24000,16000)[:,0]
with torch.inference_mode():
ref_codes=m.encode(w24).audio_codes
pad_ac=None
for cand in (False,True):
try:
ok=(Enc(m,cand).eval()(w24,w16)==ref_codes).float().mean().item()
except RuntimeError: ok=-1
print("pad_acoustic",cand,"agreement",ok)
if ok>0.99: pad_ac=cand
assert pad_ac is not None
enc=Enc(m,pad_ac).eval(); dec=Dec(m).eval()
ref_wav=m.decode(ref_codes).audio_values; print("wrapper dec == official:",(dec(ref_codes)-ref_wav).abs().max().item())
t=time.time(); torch.onnx.export(dec,(ref_codes,),f"{out}/codec_decoder.onnx",input_names=["codes"],output_names=["audio"],dynamic_axes={"codes":{2:"T"},"audio":{2:"N"}},opset_version=18,dynamo=False); print("decoder exported",round(time.time()-t,1),flush=True)
t=time.time(); torch.onnx.export(enc,(w24,w16),f"{out}/codec_encoder.onnx",input_names=["wav24","wav16"],output_names=["codes"],dynamic_axes={"wav24":{2:"N"},"wav16":{1:"N16"},"codes":{2:"T"}},opset_version=18,dynamo=False); print("encoder exported",round(time.time()-t,1),flush=True)
torch.save({"w24":w24,"w16":w16,"codes":ref_codes,"wav":ref_wav},"codec_ref.pt")
for f in ("codec_decoder.onnx","codec_encoder.onnx"): print(f, round(os.path.getsize(f"{out}/{f}")/1e6,1),"MB")
|