File size: 1,989 Bytes
5e88ff2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
from pathlib import Path

import numpy as np
import soundfile as sf
import torch
from torchaudio.functional import resample

HOP = 640
CORE, CTX = 700, 25


def load_audio(audio, sample_rate=None):
    if isinstance(audio, (str, bytes)) or hasattr(audio, '__fspath__'):
        x, sample_rate = sf.read(str(audio), dtype='float32', always_2d=True)
        return torch.from_numpy(x.mean(1)), sample_rate
    if sample_rate is None:
        raise ValueError('pass sample_rate with an array')
    x = torch.as_tensor(np.asarray(audio, dtype=np.float32))
    return (x.mean(1) if x.ndim == 2 else x), sample_rate


class SpeechTokenizer:
    def __init__(self, path, device='cuda'):
        import s3tokenizer
        path = Path(path)
        if not path.is_file():
            raise ValueError(f'The speech tokenizer is missing: put {path.name} in {path.parent}. It comes with Robin on Hugging Face.')
        self.s3, self.device = s3tokenizer, device
        self.model = s3tokenizer.load_model(str(path)).to(device).eval()

    @torch.no_grad()
    def _codes(self, wav16):
        m, lens = self.s3.padding([self.s3.log_mel_spectrogram(wav16, n_mels=128)])
        codes, clens = self.model.quantize(m.to(self.device), lens.to(self.device))
        return codes[0, :clens[0]].cpu().numpy()

    def __call__(self, audio, sample_rate=None):
        x, rate = load_audio(audio, sample_rate)
        wav16 = resample(x, rate, 16000)
        total = int(round(len(wav16) / HOP))
        if total <= CORE:
            return self._codes(wav16).astype(np.int64)
        out = []
        for s in range(0, total, CORE):
            a = max(0, s - CTX); b = min(total, s + CORE + CTX)
            c = self._codes(wav16[a * HOP:b * HOP])
            keep = min(CORE, total - s); piece = c[s - a:s - a + keep]
            if len(piece) < keep: piece = np.concatenate([piece, np.repeat(c[-1:], keep - len(piece))])
            out.append(piece)
        return np.concatenate(out).astype(np.int64)