File size: 1,989 Bytes
5e88ff2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | from pathlib import Path
import numpy as np
import soundfile as sf
import torch
from torchaudio.functional import resample
HOP = 640
CORE, CTX = 700, 25
def load_audio(audio, sample_rate=None):
if isinstance(audio, (str, bytes)) or hasattr(audio, '__fspath__'):
x, sample_rate = sf.read(str(audio), dtype='float32', always_2d=True)
return torch.from_numpy(x.mean(1)), sample_rate
if sample_rate is None:
raise ValueError('pass sample_rate with an array')
x = torch.as_tensor(np.asarray(audio, dtype=np.float32))
return (x.mean(1) if x.ndim == 2 else x), sample_rate
class SpeechTokenizer:
def __init__(self, path, device='cuda'):
import s3tokenizer
path = Path(path)
if not path.is_file():
raise ValueError(f'The speech tokenizer is missing: put {path.name} in {path.parent}. It comes with Robin on Hugging Face.')
self.s3, self.device = s3tokenizer, device
self.model = s3tokenizer.load_model(str(path)).to(device).eval()
@torch.no_grad()
def _codes(self, wav16):
m, lens = self.s3.padding([self.s3.log_mel_spectrogram(wav16, n_mels=128)])
codes, clens = self.model.quantize(m.to(self.device), lens.to(self.device))
return codes[0, :clens[0]].cpu().numpy()
def __call__(self, audio, sample_rate=None):
x, rate = load_audio(audio, sample_rate)
wav16 = resample(x, rate, 16000)
total = int(round(len(wav16) / HOP))
if total <= CORE:
return self._codes(wav16).astype(np.int64)
out = []
for s in range(0, total, CORE):
a = max(0, s - CTX); b = min(total, s + CORE + CTX)
c = self._codes(wav16[a * HOP:b * HOP])
keep = min(CORE, total - s); piece = c[s - a:s - a + keep]
if len(piece) < keep: piece = np.concatenate([piece, np.repeat(c[-1:], keep - len(piece))])
out.append(piece)
return np.concatenate(out).astype(np.int64)
|