Sparse-AST-BWM / latest /test_models.py
Chaman1234's picture
Add verified step-93213 SafeTensors release and update README
d3c3fac verified
Raw History Blame Contribute Delete
3.42 kB
"""
Model Evaluation & Comparison Suite for Sparse-AST Architectures:
Tests 3M, 10M, and 100M models on Blender Python & 3D Math Curriculum.
"""
import os
import math
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.checkpoint as cp_module
class N(nn.Module):
def __init__(s, d):
super().__init__()
s.w = nn.Parameter(torch.ones(d))
def forward(s, x):
return x * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6).to(x.dtype) * s.w
class B(nn.Module):
def __init__(s, d, h, al_hidden=24):
super().__init__()
s.n1 = N(d)
s.n2 = N(d)
s.a = nn.MultiheadAttention(d, 8, batch_first=True)
s.up = nn.Linear(d, h)
s.r = nn.Linear(h, 3)
s.down = nn.Linear(2*h, d)
s.al = nn.Sequential(nn.Linear(3*h+3, al_hidden), nn.SiLU(), nn.Linear(al_hidden, 1))
s.m = nn.Linear(d, d, bias=False)
def forward(s, x):
t = x.shape[1]
q = s.n1(x)
mask = torch.ones(t, t, device=x.device, dtype=torch.bool).triu(1)
x = torch.clamp(x + s.a(q, q, q, attn_mask=mask, need_weights=False)[0], -50, 50)
z = torch.clamp(s.up(s.n2(x)), -12, 12)
if s.training:
g = F.gumbel_softmax(s.r(z), tau=2.0, hard=True, dim=-1)
else:
# Evaluation must be deterministic; this is the legacy equivalent
# of the deterministic branch used by the corrected expansion.
best_idx = s.r(z).argmax(dim=-1, keepdim=True)
g = torch.zeros_like(z[..., :3]).scatter_(-1, best_idx, 1.0)
e = z.sign() * torch.expm1(z.abs().clamp(max=2.0))
u = (torch.stack((z, z.sign() * torch.log1p(z.abs()), e), -1) * g.unsqueeze(-2)).sum(-1).clamp(-8, 8)
p, n = F.relu(u), F.relu(-u)
d = torch.exp(-F.softplus(s.al(torch.cat((p, n, u, g), -1))).squeeze(-1)).clamp(1e-4, 0.9999)
y = torch.clamp(x + s.down(torch.cat((p, n), -1)), -50, 50)
state = torch.zeros_like(y[:, 0])
o = []
for j in range(t):
state = torch.clamp(state * d[:, j:j+1] + y[:, j] * (1 - d[:, j:j+1]), -50, 50)
o.append(torch.clamp(y[:, j] + s.m(state), -50, 50))
return torch.stack(o, 1)
class SparseAST(nn.Module):
def __init__(s, d, h, layers, al_hidden=24, seq_len=4096, vocab=512):
super().__init__()
s.d = d
s.seq_len = seq_len
s.e = nn.Embedding(vocab, d)
s.p = nn.Embedding(seq_len, d)
s.b = nn.ModuleList([B(d, h, al_hidden) for _ in range(layers)])
s.n = N(d)
s.h = nn.Linear(d, vocab, bias=False)
s.h.weight = s.e.weight
s.gradient_checkpointing = False
def gradient_checkpointing_enable(s):
s.gradient_checkpointing = True
def gradient_checkpointing_disable(s):
s.gradient_checkpointing = False
def forward(s, i):
if i.shape[1] > s.seq_len:
i = i[:, -s.seq_len:]
seq = i.shape[1]
pos = torch.arange(seq, device=i.device)[None]
x = s.e(i) + s.p(pos)
for b in s.b:
if s.training and getattr(s, 'gradient_checkpointing', False):
x = cp_module.checkpoint(b, x, use_reentrant=False)
else:
x = b(x)
return s.h(s.n(x))