Download package/robin/speech.py from FixedSeed/Robin: direct link, hf CLI and curl.
- Browser
- Download file 1.99 kB
-
https://huggingface.co/FixedSeed/Robin/resolve/main/package/robin/speech.py
- Command line
-
hf download hf://FixedSeed/Robin/package/robin/speech.py
-
curl -L -o speech.py https://huggingface.co/FixedSeed/Robin/resolve/main/package/robin/speech.py
1.99 kB
| from pathlib import Path | |
| import numpy as np | |
| import soundfile as sf | |
| import torch | |
| from torchaudio.functional import resample | |
| HOP = 640 | |
| CORE, CTX = 700, 25 | |
| def load_audio(audio, sample_rate=None): | |
| if isinstance(audio, (str, bytes)) or hasattr(audio, '__fspath__'): | |
| x, sample_rate = sf.read(str(audio), dtype='float32', always_2d=True) | |
| return torch.from_numpy(x.mean(1)), sample_rate | |
| if sample_rate is None: | |
| raise ValueError('pass sample_rate with an array') | |
| x = torch.as_tensor(np.asarray(audio, dtype=np.float32)) | |
| return (x.mean(1) if x.ndim == 2 else x), sample_rate | |
| class SpeechTokenizer: | |
| def __init__(self, path, device='cuda'): | |
| import s3tokenizer | |
| path = Path(path) | |
| if not path.is_file(): | |
| raise ValueError(f'The speech tokenizer is missing: put {path.name} in {path.parent}. It comes with Robin on Hugging Face.') | |
| self.s3, self.device = s3tokenizer, device | |
| self.model = s3tokenizer.load_model(str(path)).to(device).eval() | |
| def _codes(self, wav16): | |
| m, lens = self.s3.padding([self.s3.log_mel_spectrogram(wav16, n_mels=128)]) | |
| codes, clens = self.model.quantize(m.to(self.device), lens.to(self.device)) | |
| return codes[0, :clens[0]].cpu().numpy() | |
| def __call__(self, audio, sample_rate=None): | |
| x, rate = load_audio(audio, sample_rate) | |
| wav16 = resample(x, rate, 16000) | |
| total = int(round(len(wav16) / HOP)) | |
| if total <= CORE: | |
| return self._codes(wav16).astype(np.int64) | |
| out = [] | |
| for s in range(0, total, CORE): | |
| a = max(0, s - CTX); b = min(total, s + CORE + CTX) | |
| c = self._codes(wav16[a * HOP:b * HOP]) | |
| keep = min(CORE, total - s); piece = c[s - a:s - a + keep] | |
| if len(piece) < keep: piece = np.concatenate([piece, np.repeat(c[-1:], keep - len(piece))]) | |
| out.append(piece) | |
| return np.concatenate(out).astype(np.int64) | |