Download latest/test_models.py from Chaman1234/Sparse-AST-BWM: direct link, hf CLI and curl.
- Browser
- Download file 3.42 kB
-
https://huggingface.co/Chaman1234/Sparse-AST-BWM/resolve/main/latest/test_models.py
- Command line
-
hf download hf://Chaman1234/Sparse-AST-BWM/latest/test_models.py
-
curl -L -o test_models.py https://huggingface.co/Chaman1234/Sparse-AST-BWM/resolve/main/latest/test_models.py
3.42 kB
| """ | |
| Model Evaluation & Comparison Suite for Sparse-AST Architectures: | |
| Tests 3M, 10M, and 100M models on Blender Python & 3D Math Curriculum. | |
| """ | |
| import os | |
| import math | |
| import time | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torch.utils.checkpoint as cp_module | |
| class N(nn.Module): | |
| def __init__(s, d): | |
| super().__init__() | |
| s.w = nn.Parameter(torch.ones(d)) | |
| def forward(s, x): | |
| return x * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6).to(x.dtype) * s.w | |
| class B(nn.Module): | |
| def __init__(s, d, h, al_hidden=24): | |
| super().__init__() | |
| s.n1 = N(d) | |
| s.n2 = N(d) | |
| s.a = nn.MultiheadAttention(d, 8, batch_first=True) | |
| s.up = nn.Linear(d, h) | |
| s.r = nn.Linear(h, 3) | |
| s.down = nn.Linear(2*h, d) | |
| s.al = nn.Sequential(nn.Linear(3*h+3, al_hidden), nn.SiLU(), nn.Linear(al_hidden, 1)) | |
| s.m = nn.Linear(d, d, bias=False) | |
| def forward(s, x): | |
| t = x.shape[1] | |
| q = s.n1(x) | |
| mask = torch.ones(t, t, device=x.device, dtype=torch.bool).triu(1) | |
| x = torch.clamp(x + s.a(q, q, q, attn_mask=mask, need_weights=False)[0], -50, 50) | |
| z = torch.clamp(s.up(s.n2(x)), -12, 12) | |
| if s.training: | |
| g = F.gumbel_softmax(s.r(z), tau=2.0, hard=True, dim=-1) | |
| else: | |
| # Evaluation must be deterministic; this is the legacy equivalent | |
| # of the deterministic branch used by the corrected expansion. | |
| best_idx = s.r(z).argmax(dim=-1, keepdim=True) | |
| g = torch.zeros_like(z[..., :3]).scatter_(-1, best_idx, 1.0) | |
| e = z.sign() * torch.expm1(z.abs().clamp(max=2.0)) | |
| u = (torch.stack((z, z.sign() * torch.log1p(z.abs()), e), -1) * g.unsqueeze(-2)).sum(-1).clamp(-8, 8) | |
| p, n = F.relu(u), F.relu(-u) | |
| d = torch.exp(-F.softplus(s.al(torch.cat((p, n, u, g), -1))).squeeze(-1)).clamp(1e-4, 0.9999) | |
| y = torch.clamp(x + s.down(torch.cat((p, n), -1)), -50, 50) | |
| state = torch.zeros_like(y[:, 0]) | |
| o = [] | |
| for j in range(t): | |
| state = torch.clamp(state * d[:, j:j+1] + y[:, j] * (1 - d[:, j:j+1]), -50, 50) | |
| o.append(torch.clamp(y[:, j] + s.m(state), -50, 50)) | |
| return torch.stack(o, 1) | |
| class SparseAST(nn.Module): | |
| def __init__(s, d, h, layers, al_hidden=24, seq_len=4096, vocab=512): | |
| super().__init__() | |
| s.d = d | |
| s.seq_len = seq_len | |
| s.e = nn.Embedding(vocab, d) | |
| s.p = nn.Embedding(seq_len, d) | |
| s.b = nn.ModuleList([B(d, h, al_hidden) for _ in range(layers)]) | |
| s.n = N(d) | |
| s.h = nn.Linear(d, vocab, bias=False) | |
| s.h.weight = s.e.weight | |
| s.gradient_checkpointing = False | |
| def gradient_checkpointing_enable(s): | |
| s.gradient_checkpointing = True | |
| def gradient_checkpointing_disable(s): | |
| s.gradient_checkpointing = False | |
| def forward(s, i): | |
| if i.shape[1] > s.seq_len: | |
| i = i[:, -s.seq_len:] | |
| seq = i.shape[1] | |
| pos = torch.arange(seq, device=i.device)[None] | |
| x = s.e(i) + s.p(pos) | |
| for b in s.b: | |
| if s.training and getattr(s, 'gradient_checkpointing', False): | |
| x = cp_module.checkpoint(b, x, use_reentrant=False) | |
| else: | |
| x = b(x) | |
| return s.h(s.n(x)) | |