""" Model Evaluation & Comparison Suite for Sparse-AST Architectures: Tests 3M, 10M, and 100M models on Blender Python & 3D Math Curriculum. """ import os import math import time import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint as cp_module class N(nn.Module): def __init__(s, d): super().__init__() s.w = nn.Parameter(torch.ones(d)) def forward(s, x): return x * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6).to(x.dtype) * s.w class B(nn.Module): def __init__(s, d, h, al_hidden=24): super().__init__() s.n1 = N(d) s.n2 = N(d) s.a = nn.MultiheadAttention(d, 8, batch_first=True) s.up = nn.Linear(d, h) s.r = nn.Linear(h, 3) s.down = nn.Linear(2*h, d) s.al = nn.Sequential(nn.Linear(3*h+3, al_hidden), nn.SiLU(), nn.Linear(al_hidden, 1)) s.m = nn.Linear(d, d, bias=False) def forward(s, x): t = x.shape[1] q = s.n1(x) mask = torch.ones(t, t, device=x.device, dtype=torch.bool).triu(1) x = torch.clamp(x + s.a(q, q, q, attn_mask=mask, need_weights=False)[0], -50, 50) z = torch.clamp(s.up(s.n2(x)), -12, 12) if s.training: g = F.gumbel_softmax(s.r(z), tau=2.0, hard=True, dim=-1) else: # Evaluation must be deterministic; this is the legacy equivalent # of the deterministic branch used by the corrected expansion. best_idx = s.r(z).argmax(dim=-1, keepdim=True) g = torch.zeros_like(z[..., :3]).scatter_(-1, best_idx, 1.0) e = z.sign() * torch.expm1(z.abs().clamp(max=2.0)) u = (torch.stack((z, z.sign() * torch.log1p(z.abs()), e), -1) * g.unsqueeze(-2)).sum(-1).clamp(-8, 8) p, n = F.relu(u), F.relu(-u) d = torch.exp(-F.softplus(s.al(torch.cat((p, n, u, g), -1))).squeeze(-1)).clamp(1e-4, 0.9999) y = torch.clamp(x + s.down(torch.cat((p, n), -1)), -50, 50) state = torch.zeros_like(y[:, 0]) o = [] for j in range(t): state = torch.clamp(state * d[:, j:j+1] + y[:, j] * (1 - d[:, j:j+1]), -50, 50) o.append(torch.clamp(y[:, j] + s.m(state), -50, 50)) return torch.stack(o, 1) class SparseAST(nn.Module): def __init__(s, d, h, layers, al_hidden=24, seq_len=4096, vocab=512): super().__init__() s.d = d s.seq_len = seq_len s.e = nn.Embedding(vocab, d) s.p = nn.Embedding(seq_len, d) s.b = nn.ModuleList([B(d, h, al_hidden) for _ in range(layers)]) s.n = N(d) s.h = nn.Linear(d, vocab, bias=False) s.h.weight = s.e.weight s.gradient_checkpointing = False def gradient_checkpointing_enable(s): s.gradient_checkpointing = True def gradient_checkpointing_disable(s): s.gradient_checkpointing = False def forward(s, i): if i.shape[1] > s.seq_len: i = i[:, -s.seq_len:] seq = i.shape[1] pos = torch.arange(seq, device=i.device)[None] x = s.e(i) + s.p(pos) for b in s.b: if s.training and getattr(s, 'gradient_checkpointing', False): x = cp_module.checkpoint(b, x, use_reentrant=False) else: x = b(x) return s.h(s.n(x))