File size: 3,689 Bytes
a775b3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
"""Single-voice text-side student: width-scaled instances of Kokoro's own modules

(CustomAlbert, ProsodyPredictor, TextEncoder) with a learned constant style vector in place

of the voice input, plus a projection to the teacher's 512-channel asr space so the frozen

teacher decoder can consume its output."""
import torch, torch.nn as nn
from transformers import AlbertConfig
from kokoro.modules import CustomAlbert, ProsodyPredictor, TextEncoder

PRESETS = {  # name: (albert hidden, albert layers, albert heads, hidden_dim, style_dim, n_layer)
    # "<name>+mlp" variants use a 2-layer nonlinear asr projection (the teacher asr features are rank ~430 for 99% variance)
    "xs": (128, 4, 4, 128, 16, 2),
    "s":  (256, 6, 4, 192, 32, 3),
    "m":  (384, 6, 6, 256, 32, 3),
    "l":  (512, 8, 8, 384, 64, 3),
}

class TextStudent(nn.Module):
    def __init__(self, preset="s", n_token=178, teacher_dim=512, max_dur=50):
        super().__init__()
        mlp = preset.endswith("+mlp"); preset = preset.replace("+mlp", "")
        ah, al, an, hd, sd, nl = PRESETS[preset]
        self.bert = CustomAlbert(AlbertConfig(vocab_size=n_token, hidden_size=ah, num_attention_heads=an,
                                              intermediate_size=ah * 3, num_hidden_layers=al,
                                              max_position_embeddings=512, embedding_size=min(128, ah), dropout=0.1))
        self.bert_encoder = nn.Linear(ah, hd)
        self.predictor = ProsodyPredictor(style_dim=sd, d_hid=hd, nlayers=nl, max_dur=max_dur, dropout=0.1)
        self.text_encoder = TextEncoder(channels=hd, kernel_size=5, depth=nl, n_symbols=n_token)
        self.asr_proj = (nn.Sequential(nn.Linear(hd, teacher_dim), nn.GELU(), nn.Linear(teacher_dim, teacher_dim)) if mlp
                         else nn.Linear(hd, teacher_dim))
        self.style = nn.Parameter(torch.randn(1, sd) * 0.1)  # the one voice

    def forward(self, ids, lengths, aln=None):
        """ids [B,T] long, lengths [B]. aln: optional teacher alignment [B,T,F] (teacher forcing).

        Returns dur [B,T] (pre-round sigmoid sums), F0 [B,2F], N [B,2F], asr_tok [B,512,T]."""
        B, T = ids.shape
        mask = torch.arange(T, device=ids.device)[None].expand(B, -1) >= lengths[:, None]
        s = self.style.expand(B, -1)
        bert = self.bert(ids, attention_mask=(~mask).int())
        d_en = self.bert_encoder(bert).transpose(-1, -2)
        d = self.predictor.text_encoder(d_en, s, lengths, mask)
        x, _ = self.predictor.lstm(d)
        dur = torch.sigmoid(self.predictor.duration_proj(x)).sum(-1)
        t_en = self.text_encoder(ids, lengths, mask)
        asr_tok = self.asr_proj(t_en.transpose(-1, -2)).transpose(-1, -2)
        if aln is None:
            return dur, None, None, asr_tok
        en = d.transpose(-1, -2) @ aln
        F0, N = self.predictor.F0Ntrain(en, s)
        return dur, F0, N, asr_tok

def alignment(pred_dur, F=None):
    """pred_dur [T] long -> [T, F] one-hot expansion matrix."""
    idx = torch.repeat_interleave(torch.arange(len(pred_dur), device=pred_dur.device), pred_dur)
    F = F or len(idx)
    a = torch.zeros(len(pred_dur), F, device=pred_dur.device); a[idx[:F], torch.arange(min(F, len(idx)), device=pred_dur.device)] = 1
    return a

if __name__ == "__main__":
    for p in list(PRESETS) + ["s+mlp"]:
        m = TextStudent(p); n = sum(x.numel() for x in m.parameters())
        parts = {k: sum(x.numel() for x in getattr(m, k).parameters()) for k in ["bert", "predictor", "text_encoder", "asr_proj"]}
        print(f"{p}: {n/1e6:.2f}M  " + "  ".join(f"{k} {v/1e6:.2f}M" for k, v in parts.items()))