Robin / package /robin /speech.py
Mothersuperior's picture
Claude Opus 5.5
Robin v1
5e88ff2
Raw History Blame Contribute Delete
1.99 kB
from pathlib import Path
import numpy as np
import soundfile as sf
import torch
from torchaudio.functional import resample
HOP = 640
CORE, CTX = 700, 25
def load_audio(audio, sample_rate=None):
if isinstance(audio, (str, bytes)) or hasattr(audio, '__fspath__'):
x, sample_rate = sf.read(str(audio), dtype='float32', always_2d=True)
return torch.from_numpy(x.mean(1)), sample_rate
if sample_rate is None:
raise ValueError('pass sample_rate with an array')
x = torch.as_tensor(np.asarray(audio, dtype=np.float32))
return (x.mean(1) if x.ndim == 2 else x), sample_rate
class SpeechTokenizer:
def __init__(self, path, device='cuda'):
import s3tokenizer
path = Path(path)
if not path.is_file():
raise ValueError(f'The speech tokenizer is missing: put {path.name} in {path.parent}. It comes with Robin on Hugging Face.')
self.s3, self.device = s3tokenizer, device
self.model = s3tokenizer.load_model(str(path)).to(device).eval()
@torch.no_grad()
def _codes(self, wav16):
m, lens = self.s3.padding([self.s3.log_mel_spectrogram(wav16, n_mels=128)])
codes, clens = self.model.quantize(m.to(self.device), lens.to(self.device))
return codes[0, :clens[0]].cpu().numpy()
def __call__(self, audio, sample_rate=None):
x, rate = load_audio(audio, sample_rate)
wav16 = resample(x, rate, 16000)
total = int(round(len(wav16) / HOP))
if total <= CORE:
return self._codes(wav16).astype(np.int64)
out = []
for s in range(0, total, CORE):
a = max(0, s - CTX); b = min(total, s + CORE + CTX)
c = self._codes(wav16[a * HOP:b * HOP])
keep = min(CORE, total - s); piece = c[s - a:s - a + keep]
if len(piece) < keep: piece = np.concatenate([piece, np.repeat(c[-1:], keep - len(piece))])
out.append(piece)
return np.concatenate(out).astype(np.int64)