Download models/micro/code/student.py from owensong/Inflect-Micro-v3: direct link, hf CLI and curl.
- Browser
- Download file 3.69 kB
-
https://huggingface.co/owensong/Inflect-Micro-v3/resolve/main/models/micro/code/student.py
- Command line
-
hf download hf://owensong/Inflect-Micro-v3/models/micro/code/student.py
-
curl -L -o student.py https://huggingface.co/owensong/Inflect-Micro-v3/resolve/main/models/micro/code/student.py
3.69 kB
| """Single-voice text-side student: width-scaled instances of Kokoro's own modules | |
| (CustomAlbert, ProsodyPredictor, TextEncoder) with a learned constant style vector in place | |
| of the voice input, plus a projection to the teacher's 512-channel asr space so the frozen | |
| teacher decoder can consume its output.""" | |
| import torch, torch.nn as nn | |
| from transformers import AlbertConfig | |
| from kokoro.modules import CustomAlbert, ProsodyPredictor, TextEncoder | |
| PRESETS = { # name: (albert hidden, albert layers, albert heads, hidden_dim, style_dim, n_layer) | |
| # "<name>+mlp" variants use a 2-layer nonlinear asr projection (the teacher asr features are rank ~430 for 99% variance) | |
| "xs": (128, 4, 4, 128, 16, 2), | |
| "s": (256, 6, 4, 192, 32, 3), | |
| "m": (384, 6, 6, 256, 32, 3), | |
| "l": (512, 8, 8, 384, 64, 3), | |
| } | |
| class TextStudent(nn.Module): | |
| def __init__(self, preset="s", n_token=178, teacher_dim=512, max_dur=50): | |
| super().__init__() | |
| mlp = preset.endswith("+mlp"); preset = preset.replace("+mlp", "") | |
| ah, al, an, hd, sd, nl = PRESETS[preset] | |
| self.bert = CustomAlbert(AlbertConfig(vocab_size=n_token, hidden_size=ah, num_attention_heads=an, | |
| intermediate_size=ah * 3, num_hidden_layers=al, | |
| max_position_embeddings=512, embedding_size=min(128, ah), dropout=0.1)) | |
| self.bert_encoder = nn.Linear(ah, hd) | |
| self.predictor = ProsodyPredictor(style_dim=sd, d_hid=hd, nlayers=nl, max_dur=max_dur, dropout=0.1) | |
| self.text_encoder = TextEncoder(channels=hd, kernel_size=5, depth=nl, n_symbols=n_token) | |
| self.asr_proj = (nn.Sequential(nn.Linear(hd, teacher_dim), nn.GELU(), nn.Linear(teacher_dim, teacher_dim)) if mlp | |
| else nn.Linear(hd, teacher_dim)) | |
| self.style = nn.Parameter(torch.randn(1, sd) * 0.1) # the one voice | |
| def forward(self, ids, lengths, aln=None): | |
| """ids [B,T] long, lengths [B]. aln: optional teacher alignment [B,T,F] (teacher forcing). | |
| Returns dur [B,T] (pre-round sigmoid sums), F0 [B,2F], N [B,2F], asr_tok [B,512,T].""" | |
| B, T = ids.shape | |
| mask = torch.arange(T, device=ids.device)[None].expand(B, -1) >= lengths[:, None] | |
| s = self.style.expand(B, -1) | |
| bert = self.bert(ids, attention_mask=(~mask).int()) | |
| d_en = self.bert_encoder(bert).transpose(-1, -2) | |
| d = self.predictor.text_encoder(d_en, s, lengths, mask) | |
| x, _ = self.predictor.lstm(d) | |
| dur = torch.sigmoid(self.predictor.duration_proj(x)).sum(-1) | |
| t_en = self.text_encoder(ids, lengths, mask) | |
| asr_tok = self.asr_proj(t_en.transpose(-1, -2)).transpose(-1, -2) | |
| if aln is None: | |
| return dur, None, None, asr_tok | |
| en = d.transpose(-1, -2) @ aln | |
| F0, N = self.predictor.F0Ntrain(en, s) | |
| return dur, F0, N, asr_tok | |
| def alignment(pred_dur, F=None): | |
| """pred_dur [T] long -> [T, F] one-hot expansion matrix.""" | |
| idx = torch.repeat_interleave(torch.arange(len(pred_dur), device=pred_dur.device), pred_dur) | |
| F = F or len(idx) | |
| a = torch.zeros(len(pred_dur), F, device=pred_dur.device); a[idx[:F], torch.arange(min(F, len(idx)), device=pred_dur.device)] = 1 | |
| return a | |
| if __name__ == "__main__": | |
| for p in list(PRESETS) + ["s+mlp"]: | |
| m = TextStudent(p); n = sum(x.numel() for x in m.parameters()) | |
| parts = {k: sum(x.numel() for x in getattr(m, k).parameters()) for k in ["bert", "predictor", "text_encoder", "asr_proj"]} | |
| print(f"{p}: {n/1e6:.2f}M " + " ".join(f"{k} {v/1e6:.2f}M" for k, v in parts.items())) | |