Spaces:
Running
Running
Download frontend/src/content/code.ts from philippyt/ablation-lab: direct link, hf CLI and curl.
- Browser
- Download file 13.4 kB
-
https://huggingface.co/spaces/philippyt/ablation-lab/resolve/main/frontend/src/content/code.ts
- Command line
-
hf download hf://spaces/philippyt/ablation-lab/frontend/src/content/code.ts
-
curl -L -o code.ts https://huggingface.co/spaces/philippyt/ablation-lab/resolve/main/frontend/src/content/code.ts
13.4 kB
| import type { ArchConfig } from "../types"; | |
| function pyBool(b: boolean): string { | |
| return b ? "True" : "False"; | |
| } | |
| function pyStr(s: string): string { | |
| return `"${s}"`; | |
| } | |
| function configBlock(cfg: ArchConfig): string { | |
| return `@dataclass | |
| class Config: | |
| vocab_size: int = ${cfg.vocab_size} | |
| d_model: int = ${cfg.d_model} | |
| n_layers: int = ${cfg.n_layers} | |
| n_heads: int = ${cfg.n_heads} | |
| n_kv_heads: int = ${cfg.n_kv_heads} | |
| head_dim: int = ${cfg.head_dim} | |
| ctx: int = ${cfg.ctx} | |
| ffn_mult: int = ${cfg.ffn_mult} | |
| n_experts: int = ${cfg.n_experts} | |
| top_k: int = ${cfg.top_k} | |
| tie_embeddings: bool = ${pyBool(cfg.tie_embeddings)} | |
| pos_embed: str = ${pyStr(cfg.pos_embed)} | |
| activation: str = ${pyStr(cfg.activation)} | |
| norm: str = ${pyStr(cfg.norm)} | |
| ffn: str = ${pyStr(cfg.ffn)} | |
| qk_norm: bool = ${pyBool(cfg.qk_norm)} | |
| attention_type: str = ${pyStr(cfg.attention_type)} | |
| enable_attention: bool = ${pyBool(cfg.enable_attention)} | |
| enable_ffn: bool = ${pyBool(cfg.enable_ffn)} | |
| enable_norms: bool = ${pyBool(cfg.enable_norms)} | |
| def __post_init__(self): | |
| if self.head_dim * self.n_heads != self.d_model: | |
| assert self.d_model % self.n_heads == 0, "d_model must be divisible by n_heads" | |
| self.head_dim = self.d_model // self.n_heads | |
| assert self.head_dim * self.n_heads == self.d_model, "head_dim * n_heads must equal d_model" | |
| assert self.head_dim % 2 == 0, "head_dim must be even (RoPE and MLA split require it)" | |
| assert self.n_kv_heads >= 1 and self.n_heads % self.n_kv_heads == 0, "n_heads must be a positive multiple of n_kv_heads" | |
| if self.ffn == "moe": | |
| assert self.top_k >= 1 and self.top_k <= self.n_experts, "top_k must be in [1, n_experts]"`; | |
| } | |
| const MODULES = `class RMSNorm(nn.Module): | |
| def __init__(self, d, eps=1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(d)) | |
| def forward(self, x): | |
| rms = x.pow(2).mean(-1, keepdim=True).add(self.eps).rsqrt() | |
| return x * rms * self.weight | |
| class PerHeadRMSNorm(nn.Module): | |
| def __init__(self, n_heads, head_dim, eps=1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(n_heads, head_dim)) | |
| def forward(self, x): | |
| rms = x.pow(2).mean(-1, keepdim=True).add(self.eps).rsqrt() | |
| return x * rms * self.weight | |
| def make_norm(cfg): | |
| return nn.LayerNorm(cfg.d_model) if cfg.norm == "layernorm" else RMSNorm(cfg.d_model) | |
| def activation_fn(kind): | |
| if kind == "silu": | |
| return F.silu | |
| if kind == "gelu": | |
| return F.gelu | |
| return lambda z: F.relu(z).pow(2) | |
| def ffn_hidden(cfg): | |
| h = int((2 / 3) * cfg.ffn_mult * cfg.d_model) | |
| return ((h + 63) // 64) * 64 | |
| def build_rope_cache(head_dim, max_seq, base=10000.0): | |
| half = head_dim // 2 | |
| freqs = 1.0 / (base ** (torch.arange(0, half).float() / half)) | |
| t = torch.arange(max_seq).float() | |
| angles = torch.outer(t, freqs) | |
| return angles.cos(), angles.sin() | |
| def apply_rope(x, cos, sin): | |
| x1, x2 = x.chunk(2, dim=-1) | |
| cos = cos.unsqueeze(0).unsqueeze(0) | |
| sin = sin.unsqueeze(0).unsqueeze(0) | |
| return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1) | |
| def sinusoidal_table(ctx, d_model): | |
| pos = torch.arange(ctx).unsqueeze(1).float() | |
| div = torch.exp(torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)) | |
| pe = torch.zeros(ctx, d_model) | |
| pe[:, 0::2] = torch.sin(pos * div) | |
| pe[:, 1::2] = torch.cos(pos * div) | |
| return pe | |
| def alibi_slopes(n_heads): | |
| return torch.tensor([2 ** (-8 * (i + 1) / n_heads) for i in range(n_heads)]) | |
| def alibi_bias_matrix(seq_len, slopes, device, dtype): | |
| i = torch.arange(seq_len, device=device).unsqueeze(1) | |
| j = torch.arange(seq_len, device=device).unsqueeze(0) | |
| distance = (j - i).clamp(max=0).to(dtype) | |
| return slopes.to(device=device, dtype=dtype).view(-1, 1, 1) * distance | |
| class Attention(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| d, H, KV, dh = cfg.d_model, cfg.n_heads, cfg.n_kv_heads, cfg.head_dim | |
| self.group_size = H // KV | |
| self.q_proj = nn.Linear(d, H * dh, bias=False) | |
| self.k_proj = nn.Linear(d, KV * dh, bias=False) | |
| self.v_proj = nn.Linear(d, KV * dh, bias=False) | |
| self.o_proj = nn.Linear(H * dh, d, bias=False) | |
| if cfg.pos_embed == "rope": | |
| cos, sin = build_rope_cache(dh, cfg.ctx) | |
| self.register_buffer("rope_cos", cos, persistent=False) | |
| self.register_buffer("rope_sin", sin, persistent=False) | |
| if cfg.pos_embed == "alibi": | |
| self.register_buffer("alibi_slopes", alibi_slopes(H), persistent=False) | |
| if cfg.qk_norm: | |
| self.q_norm = PerHeadRMSNorm(H, dh) | |
| self.k_norm = PerHeadRMSNorm(KV, dh) | |
| else: | |
| self.q_norm = None | |
| self.k_norm = None | |
| def forward(self, x): | |
| cfg = self.cfg | |
| B, T, _ = x.shape | |
| H, KV, dh = cfg.n_heads, cfg.n_kv_heads, cfg.head_dim | |
| q = self.q_proj(x).view(B, T, H, dh).transpose(1, 2) | |
| k = self.k_proj(x).view(B, T, KV, dh).transpose(1, 2) | |
| v = self.v_proj(x).view(B, T, KV, dh).transpose(1, 2) | |
| if self.q_norm is not None: | |
| q = self.q_norm(q.transpose(1, 2)).transpose(1, 2) | |
| k = self.k_norm(k.transpose(1, 2)).transpose(1, 2) | |
| if cfg.pos_embed == "rope": | |
| q = apply_rope(q, self.rope_cos[:T], self.rope_sin[:T]) | |
| k = apply_rope(k, self.rope_cos[:T], self.rope_sin[:T]) | |
| if self.group_size != 1: | |
| k = k.repeat_interleave(self.group_size, dim=1) | |
| v = v.repeat_interleave(self.group_size, dim=1) | |
| if cfg.pos_embed == "alibi": | |
| alibi = alibi_bias_matrix(T, self.alibi_slopes, x.device, x.dtype) | |
| causal = torch.full((T, T), float("-inf"), device=x.device, dtype=x.dtype).triu(1) | |
| mask = causal.unsqueeze(0) + alibi | |
| out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=False) | |
| else: | |
| out = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| out = out.transpose(1, 2).contiguous().view(B, T, H * dh) | |
| return self.o_proj(out) | |
| class MLAAttention(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| d, H, dh = cfg.d_model, cfg.n_heads, cfg.head_dim | |
| self.rope_dim = dh // 2 | |
| self.nope_dim = dh - self.rope_dim | |
| self.latent = max(8, d // 8) | |
| self.q_proj = nn.Linear(d, H * dh, bias=False) | |
| self.kv_down = nn.Linear(d, self.latent, bias=False) | |
| self.k_rope_proj = nn.Linear(d, self.rope_dim, bias=False) | |
| self.k_up = nn.Linear(self.latent, H * self.nope_dim, bias=False) | |
| self.v_up = nn.Linear(self.latent, H * dh, bias=False) | |
| self.o_proj = nn.Linear(H * dh, d, bias=False) | |
| cos, sin = build_rope_cache(self.rope_dim, cfg.ctx) | |
| self.register_buffer("rope_cos", cos, persistent=False) | |
| self.register_buffer("rope_sin", sin, persistent=False) | |
| if cfg.qk_norm: | |
| self.q_norm = PerHeadRMSNorm(H, dh) | |
| self.k_norm = PerHeadRMSNorm(H, dh) | |
| else: | |
| self.q_norm = None | |
| self.k_norm = None | |
| def forward(self, x): | |
| cfg = self.cfg | |
| B, T, _ = x.shape | |
| H, dh = cfg.n_heads, cfg.head_dim | |
| q = self.q_proj(x).view(B, T, H, dh).transpose(1, 2) | |
| q_nope, q_rope = q.split([self.nope_dim, self.rope_dim], dim=-1) | |
| latent = self.kv_down(x) | |
| k_rope = self.k_rope_proj(x).unsqueeze(2) | |
| cos, sin = self.rope_cos[:T], self.rope_sin[:T] | |
| q_rope_rot = apply_rope(q_rope, cos, sin) | |
| k_rope_rot = apply_rope(k_rope.transpose(1, 2), cos, sin).transpose(1, 2) | |
| k_nope = self.k_up(latent).view(B, T, H, self.nope_dim).transpose(1, 2) | |
| v = self.v_up(latent).view(B, T, H, dh).transpose(1, 2) | |
| k_rope_full = k_rope_rot.repeat(1, 1, H, 1).transpose(1, 2) | |
| k = torch.cat([k_nope, k_rope_full], dim=-1).contiguous() | |
| q_full = torch.cat([q_nope, q_rope_rot], dim=-1).contiguous() | |
| if self.q_norm is not None: | |
| q_full = self.q_norm(q_full.transpose(1, 2)).transpose(1, 2) | |
| k = self.k_norm(k.transpose(1, 2)).transpose(1, 2) | |
| out = F.scaled_dot_product_attention(q_full, k, v, is_causal=True) | |
| out = out.transpose(1, 2).contiguous().view(B, T, H * dh) | |
| return self.o_proj(out) | |
| def make_attention(cfg): | |
| return MLAAttention(cfg) if cfg.attention_type == "mla" else Attention(cfg) | |
| class GatedFFN(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| d = cfg.d_model | |
| h = ffn_hidden(cfg) | |
| self.w_gate = nn.Linear(d, h, bias=False) | |
| self.w_up = nn.Linear(d, h, bias=False) | |
| self.w_down = nn.Linear(h, d, bias=False) | |
| self.act = activation_fn(cfg.activation) | |
| def forward(self, x): | |
| return self.w_down(self.act(self.w_gate(x)) * self.w_up(x)) | |
| class MoE(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.gate = nn.Linear(cfg.d_model, cfg.n_experts, bias=False) | |
| self.experts = nn.ModuleList([GatedFFN(cfg) for _ in range(cfg.n_experts)]) | |
| def forward(self, x): | |
| cfg = self.cfg | |
| B, T, D = x.shape | |
| x_flat = x.reshape(B * T, D) | |
| probs = F.softmax(self.gate(x_flat), dim=-1) | |
| top_w, top_idx = probs.topk(cfg.top_k, dim=-1) | |
| top_w = top_w / top_w.sum(-1, keepdim=True) | |
| out = torch.zeros_like(x_flat) | |
| for e in range(cfg.n_experts): | |
| mask = (top_idx == e) | |
| if not mask.any(): | |
| continue | |
| tok = mask.any(-1).nonzero(as_tuple=True)[0] | |
| w = (mask[tok].to(top_w.dtype) * top_w[tok]).sum(-1, keepdim=True) | |
| out[tok] = out[tok] + w * self.experts[e](x_flat[tok]) | |
| return out.view(B, T, D) | |
| def make_ffn(cfg): | |
| return MoE(cfg) if cfg.ffn == "moe" else GatedFFN(cfg) | |
| class Block(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.norm1 = make_norm(cfg) | |
| self.attn = make_attention(cfg) | |
| self.norm2 = make_norm(cfg) | |
| self.ffn = make_ffn(cfg) | |
| def forward(self, x): | |
| cfg = self.cfg | |
| if cfg.enable_attention: | |
| x_in = self.norm1(x) if cfg.enable_norms else x | |
| x = x + self.attn(x_in) | |
| if cfg.enable_ffn: | |
| x_in = self.norm2(x) if cfg.enable_norms else x | |
| x = x + self.ffn(x_in) | |
| return x | |
| class TinyTransformer(nn.Module): | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.cfg = cfg | |
| self.tok_embed = nn.Embedding(cfg.vocab_size, cfg.d_model) | |
| if cfg.pos_embed == "absolute": | |
| self.abs_pos = nn.Parameter(torch.zeros(cfg.ctx, cfg.d_model)) | |
| else: | |
| self.abs_pos = None | |
| if cfg.pos_embed == "sinusoidal": | |
| self.register_buffer("sin_pe", sinusoidal_table(cfg.ctx, cfg.d_model), persistent=False) | |
| self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layers)]) | |
| self.final_norm = make_norm(cfg) | |
| if cfg.tie_embeddings: | |
| self.lm_head = None | |
| else: | |
| self.lm_head = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False) | |
| def forward(self, idx): | |
| cfg = self.cfg | |
| T = idx.size(1) | |
| x = self.tok_embed(idx) | |
| if self.abs_pos is not None and cfg.pos_embed == "absolute": | |
| x = x + self.abs_pos[:T].unsqueeze(0) | |
| elif cfg.pos_embed == "sinusoidal": | |
| x = x + self.sin_pe[:T].unsqueeze(0) | |
| for block in self.blocks: | |
| x = block(x) | |
| if cfg.enable_norms: | |
| x = self.final_norm(x) | |
| if self.lm_head is None: | |
| logits = x @ self.tok_embed.weight.t() | |
| else: | |
| logits = self.lm_head(x) | |
| return logits`; | |
| const STUB = `cfg = Config() | |
| model = TinyTransformer(cfg).eval() | |
| T = min(16, cfg.ctx) | |
| idx = torch.randint(0, cfg.vocab_size, (2, T)) | |
| with torch.no_grad(): | |
| out = model(idx) | |
| print("output shape:", tuple(out.shape))`; | |
| const IMPORTS = `import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from dataclasses import dataclass`; | |
| export function composeCode(cfg: ArchConfig): string { | |
| return [ | |
| IMPORTS, | |
| configBlock(cfg), | |
| MODULES, | |
| STUB, | |
| ].join("\n\n").replace(/\n\n\n+/g, "\n\n"); | |
| } | |
| const PY_KW = ["def","class","return","if","elif","else","import","from","as","for","in","while","not","and","or","is","with","lambda","None","True","False","self","super","torch","nn","F","math","int","str","bool","assert"]; | |
| export function highlightPython(code: string): string { | |
| const escaped = code.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">"); | |
| const kwAlt = PY_KW.join("|"); | |
| const tokenRe = new RegExp( | |
| `(#[^\\n]*)` + | |
| `|("[^"\\n]*")` + | |
| `|(\\b(?:${kwAlt})\\b)` + | |
| `|(\\b\\d+\\.?\\d*\\b)`, | |
| "g" | |
| ); | |
| return escaped.replace(tokenRe, (m, c, s, k, n) => { | |
| if (c) return `<span class="py-comment">${c}</span>`; | |
| if (s) return `<span class="py-str">${s}</span>`; | |
| if (k) return `<span class="py-kw">${k}</span>`; | |
| if (n) return `<span class="py-num">${n}</span>`; | |
| return m; | |
| }); | |
| } | |