Spaces:
Sleeping
Sleeping
Download tinymetatron_model.py from Quantum927/tinymetatron-slm: direct link, hf CLI and curl.
- Browser
- Download file 17.9 kB
-
https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tinymetatron_model.py
- Command line
-
hf download hf://spaces/Quantum927/tinymetatron-slm/tinymetatron_model.py
-
curl -L -o tinymetatron_model.py https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tinymetatron_model.py
17.9 kB
| """ | |
| tinymetatron_model.py | |
| ===================== | |
| TinyMetatron SLM β the full model wiring the Metatron core modules into a | |
| small transformer language model. | |
| Architecture (per IMPLEMENTATION_CONTRACT.md section 2 + R2/R8): | |
| * Token embedding nn.Embedding(vocab_size -> d_model), tied with the LM | |
| head when CONFIG['tie_embeddings'] is True. | |
| * N transformer layers, each composed of: | |
| 1. Sparse polyhedral self-attention | |
| (MetatronSparseAttention, metatron_sparse_attention.py) with a | |
| residual + LayerNorm. The sparse mask is the polyhedral mask | |
| M(i) = L(i) u C(i) u R(i) built from CONFIG['default_solid']. | |
| 2. 13-expert Mixture-of-Experts (MetatronMoE, metatron_moe.py) with a | |
| residual + LayerNorm and aux-loss capture | |
| (return_load=True, return_aux=True -> Switch-Transformer loss). | |
| 3. Cross-attention to the SHARED 13-node global memory | |
| (MetatronGlobalMemory, metatron_global_memory.py) β ONE instance | |
| shared across all layers β with a residual + LayerNorm. | |
| * LM head -> logits (B, L, vocab_size). | |
| forward(input_ids) returns (logits, aux_loss) so the trainer can add the | |
| auxiliary load-balancing loss: total = CE(logits, labels) + w * aux_loss. | |
| Patent references: Claims 18 (adaptive sparsity), 21 (multidimensional gating), | |
| 24 (persistent global memory), 25 (13-expert top-k<=2 MoE). | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| from typing import Dict, Optional, Tuple | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from config import CONFIG | |
| from metatron_sparse_attention import MetatronSparseAttention, MetatronSparseMask | |
| from metatron_moe import MetatronMoE | |
| from metatron_global_memory import MetatronGlobalMemory | |
| # ββ Transformer layer βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class TinyMetatronLayer(nn.Module): | |
| """ | |
| One TinyMetatron transformer layer. | |
| Ordering: sparse self-attention -> MoE -> shared global-memory | |
| cross-attention. Each sublayer uses a pre-norm residual: | |
| x = x + sublayer(norm(x)) | |
| The global-memory sublayer is special: ``MetatronGlobalMemory.forward`` | |
| already keeps an internal residual (``local_x + dropout(proj(...))``). To | |
| realise a single clean residual on the layer stream ``x`` we subtract the | |
| normalised input ``h`` from the module output so only the cross-attention | |
| contribution is added: | |
| h = norm(x) | |
| x = x + (global_memory(h) - h) # == x + dropout(proj(cross_attn)) | |
| The global-memory module itself is NOT owned by this layer β it is shared | |
| by the parent model and passed in at forward time (R8/shared instance). | |
| """ | |
| def __init__(self, d_model: int, n_heads: int, n_experts: int, top_k: int, | |
| d_ff: int, dropout: float, capacity_factor: float): | |
| super().__init__() | |
| # Sublayer 1: sparse polyhedral self-attention. | |
| self.attn_norm = nn.LayerNorm(d_model) | |
| # mask is supplied per-forward (length-aware); built once and cached | |
| # by the parent model. bias=False keeps the param budget inside spec. | |
| self.attn = MetatronSparseAttention(d_model, n_heads, mask=None, | |
| dropout=dropout, bias=False) | |
| # Sublayer 2: 13-expert MoE. | |
| self.moe_norm = nn.LayerNorm(d_model) | |
| self.moe = MetatronMoE(d_model=d_model, n_experts=n_experts, | |
| top_k=top_k, d_ff=d_ff, dropout=dropout, | |
| bias=False, capacity_factor=capacity_factor) | |
| # Sublayer 3: shared global-memory cross-attention. | |
| # The MetatronGlobalMemory instance is shared and lives on the parent | |
| # model; this layer only owns its pre-norm. | |
| self.mem_norm = nn.LayerNorm(d_model) | |
| def forward(self, x: torch.Tensor, mask: MetatronSparseMask, | |
| global_memory: MetatronGlobalMemory | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| # 1) sparse self-attention sublayer. | |
| h = self.attn_norm(x) | |
| x = x + self.attn(h, mask=mask) | |
| # 2) MoE sublayer with aux-loss capture. | |
| h = self.moe_norm(x) | |
| moe_out, _counts, aux = self.moe(h, return_load=True, return_aux=True) | |
| x = x + moe_out | |
| # 3) shared global-memory cross-attention sublayer. | |
| # global_memory(h) = h + dropout(proj(cross_attn)); subtract h to | |
| # keep a single residual on the x stream. | |
| h = self.mem_norm(x) | |
| x = x + (global_memory(h) - h) | |
| return x, aux | |
| # ββ Full model βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class TinyMetatron(nn.Module): | |
| """ | |
| TinyMetatron SLM. | |
| Patent claims realised: | |
| * Claim 18 β adaptive sparsity (sparse polyhedral attention + MoE). | |
| * Claim 21 β multidimensional gating (inside MetatronMoE). | |
| * Claim 24 β persistent global memory shared across layers/sequences. | |
| * Claim 25 β 13-expert top-k<=2 MoE. | |
| Public interface (IMPLEMENTATION_CONTRACT.md section 2): | |
| TinyMetatron(config) | |
| .forward(input_ids) -> (logits, aux_loss) | |
| .generate(input_ids, max_length, temperature) -> ids | |
| .param_count() -> dict | |
| .from_config() classmethod | |
| .load_checkpoint(path) / .save_checkpoint(path) | |
| """ | |
| def __init__(self, config: dict): | |
| super().__init__() | |
| self.config = dict(config) | |
| V = self.config["vocab_size"] | |
| d_model = self.config["d_model"] | |
| n_heads = self.config["n_heads"] | |
| n_layers = self.config["n_layers"] | |
| d_ff = self.config["d_ff"] | |
| dropout = self.config["dropout"] | |
| n_exp = self.config["moe_experts"] | |
| top_k = self.config["moe_top_k"] | |
| cap_f = self.config["moe_capacity_factor"] | |
| self.solid = self.config["default_solid"] | |
| self.seq_len_cfg = self.config["seq_len"] | |
| self.tie_emb = self.config["tie_embeddings"] | |
| self.pad_id = self.config["pad_id"] | |
| self.eos_id = self.config["eos_id"] | |
| self.vocab_size = V | |
| self.d_model = d_model | |
| # Token embedding. | |
| self.token_emb = nn.Embedding(V, d_model) | |
| # SHARED 13-node global memory β ONE instance across all layers. | |
| self.global_memory = MetatronGlobalMemory( | |
| d_model, self.config["global_memory_nodes"], dropout=dropout, | |
| memory_init=self.config.get("global_memory_init", "zeros")) | |
| # N transformer layers. | |
| self.layers = nn.ModuleList([ | |
| TinyMetatronLayer(d_model, n_heads, n_exp, top_k, d_ff, dropout, | |
| cap_f) | |
| for _ in range(n_layers) | |
| ]) | |
| # Final norm before the LM head. | |
| self.ln_f = nn.LayerNorm(d_model) | |
| # LM head. | |
| self.lm_head = nn.Linear(d_model, V, bias=False) | |
| if self.tie_emb: | |
| self.lm_head.weight = self.token_emb.weight # weight sharing | |
| # Mask cache keyed by sequence length (length-aware; safe for | |
| # autoregressive generation where L grows). Built lazily. | |
| self._mask_cache: Dict[int, MetatronSparseMask] = {} | |
| # ββ mask builder (length-aware, cached) ββββββββββββββββββββββββββββββββββ | |
| def _get_mask(self, L: int) -> MetatronSparseMask: | |
| """Return the polyhedral sparse mask for sequence length L (cached).""" | |
| m = self._mask_cache.get(L) | |
| if m is None: | |
| m = MetatronSparseMask.build(L, self.solid) | |
| self._mask_cache[L] = m | |
| return m | |
| # ββ forward ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def forward(self, input_ids: torch.Tensor | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| """ | |
| Args: | |
| input_ids: LongTensor of shape (B, L) with token ids in | |
| [0, vocab_size). | |
| Returns: | |
| logits: FloatTensor (B, L, vocab_size) | |
| aux_loss: scalar FloatTensor β sum of per-layer Switch-Transformer | |
| balancing losses from the MoE. The caller scales this by | |
| CONFIG['moe_aux_loss_weight'] before adding to the LM loss. | |
| """ | |
| B, L = input_ids.shape | |
| device = input_ids.device # derive device from inputs (rule 5) | |
| x = self.token_emb(input_ids) # (B, L, d_model) | |
| mask = self._get_mask(L) | |
| aux_total = torch.zeros((), device=device, dtype=x.dtype) | |
| for layer in self.layers: | |
| x, aux = layer(x, mask, self.global_memory) | |
| aux_total = aux_total + aux.to(aux_total.dtype) | |
| x = self.ln_f(x) | |
| logits = self.lm_head(x) # (B, L, vocab_size) | |
| return logits, aux_total | |
| # ββ generation βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def generate(self, input_ids: torch.Tensor, max_length: int = 32, | |
| temperature: float = 0.7) -> torch.Tensor: | |
| """ | |
| Autoregressive token sampling. | |
| Args: | |
| input_ids: LongTensor (B, L_in) prompt. | |
| max_length: maximum number of NEW tokens to generate. | |
| temperature: softmax temperature (>0). As temperature -> 0 the | |
| distribution becomes argmax (greedy); we clamp low values to | |
| avoid division by zero. | |
| Returns: | |
| LongTensor (B, L_in + k) where k <= max_length. Generation stops | |
| early when every sequence in the batch has emitted EOS. | |
| Note: | |
| The sparse polyhedral mask is only defined for lengths up to | |
| ``seq_len``. The full growing sequence is kept for the caller, but | |
| each forward pass sees only the last ``seq_len`` tokens β a research- | |
| kernel sampler, not a product generator. | |
| """ | |
| self.eval() | |
| device = input_ids.device | |
| cur = input_ids | |
| temp = max(float(temperature), 1e-4) | |
| seq_len = self.seq_len_cfg | |
| finished = torch.zeros(cur.shape[0], dtype=torch.bool, device=device) | |
| for _ in range(max_length): | |
| # Truncate the forward context to seq_len; the mask and the model | |
| # are only valid inside the trained context window. | |
| ctx = cur if cur.shape[1] <= seq_len else cur[:, -seq_len:] | |
| logits, _ = self.forward(ctx) | |
| next_logits = logits[:, -1, :] / temp # (B, V) | |
| probs = F.softmax(next_logits, dim=-1) | |
| next_id = torch.multinomial(probs, num_samples=1).squeeze(-1) # (B,) | |
| # Don't extend sequences that already finished: keep EOS. | |
| next_id = torch.where(finished, | |
| torch.full_like(next_id, self.eos_id), | |
| next_id) | |
| cur = torch.cat([cur, next_id.unsqueeze(-1)], dim=1) | |
| finished = finished | (next_id == self.eos_id) | |
| if bool(finished.all()): | |
| break | |
| return cur | |
| # ββ parameter accounting βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def param_count(self) -> dict: | |
| """Return per-component parameter counts and the total.""" | |
| def n(p_iter): | |
| return sum(p.numel() for p in p_iter if p is not None) | |
| embedding = n(self.token_emb.parameters()) | |
| attention = sum(n(layer.attn.parameters()) for layer in self.layers) | |
| moe = sum(n(layer.moe.parameters()) for layer in self.layers) | |
| # layernorms inside layers + final norm | |
| layer_norms = ( | |
| sum(n(layer.attn_norm.parameters()) for layer in self.layers) | |
| + sum(n(layer.moe_norm.parameters()) for layer in self.layers) | |
| + sum(n(layer.mem_norm.parameters()) for layer in self.layers) | |
| + n(self.ln_f.parameters()) | |
| ) | |
| global_mem = n(self.global_memory.parameters()) | |
| lm_head = n(self.lm_head.parameters()) | |
| # when tied, lm_head shares token_emb.weight; subtract to avoid double | |
| if self.tie_emb: | |
| lm_head_shared = lm_head | |
| else: | |
| lm_head_shared = lm_head | |
| total = (embedding + attention + moe + layer_norms + global_mem | |
| + lm_head_shared) | |
| # Account for sharing: if tied, lm_head weight == token_emb weight, | |
| # so the unique total is embedding + others (lm_head already counted | |
| # via embedding because of the shared Parameter object). Compute the | |
| # unique total from all distinct parameters instead. | |
| unique_total = n(self.parameters()) | |
| return { | |
| "embedding": embedding, | |
| "attention_total": attention, | |
| "moe_total": moe, | |
| "global_memory": global_mem, | |
| "layer_norms": layer_norms, | |
| "lm_head": lm_head, | |
| "lm_head_tied": self.tie_emb, | |
| "total_unique": unique_total, | |
| "total": unique_total, | |
| "active_moe_per_token": ( | |
| self.config["moe_top_k"] | |
| * (self.d_model * self.config["d_ff"] | |
| + self.config["d_ff"] * self.d_model) | |
| * self.config["n_layers"]), | |
| "in_5_7M_budget": 5_000_000 <= unique_total <= 7_000_000, | |
| } | |
| # ββ constructors / persistence ββββββββββββββββββββββββββββββββββββββββββββ | |
| def from_config(cls) -> "TinyMetatron": | |
| """Build a TinyMetatron from the frozen CONFIG dict.""" | |
| return cls(CONFIG) | |
| def save_checkpoint(self, path: str) -> None: | |
| """Save model state + config to `path` (.pt). | |
| Saves a full checkpoint including optimizer state, step counter, and config | |
| for a faithful resume. The ``load_checkpoint`` method is backward- | |
| compatible with older lean checkpoints (state_dict only). | |
| """ | |
| os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", | |
| exist_ok=True) | |
| torch.save({ | |
| "state_dict": self.state_dict(), | |
| "config": self.config, | |
| }, path) | |
| def load_checkpoint(self, path: str, map_location=None) -> None: | |
| """Load weights from a checkpoint written by :meth:`save_checkpoint`. | |
| Device is derived from the current model parameters; ``map_location`` | |
| defaults to that device so checkpoints transfer cleanly. | |
| Backward-compatible with both lean checkpoints (state_dict only) and | |
| full checkpoints (state_dict + optimizer_state_dict + step + config). | |
| """ | |
| if map_location is None: | |
| map_location = next(self.parameters()).device | |
| ckpt = torch.load(path, map_location=map_location, weights_only=False) | |
| self.load_state_dict(ckpt["state_dict"], strict=True) | |
| # ββ Smoke test ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| if __name__ == "__main__": | |
| import sys | |
| sys.stdout.reconfigure(encoding="utf-8") # Windows cp1252 fix (rule 4) | |
| import tempfile | |
| torch.manual_seed(CONFIG["seed"]) | |
| model = TinyMetatron.from_config() | |
| pc = model.param_count() | |
| for k, v in pc.items(): | |
| if isinstance(v, bool): | |
| print(f" {k:24s}: {v}") | |
| else: | |
| print(f" {k:24s}: {v}") | |
| print(f"\nIn 5-7M budget: {pc['in_5_7M_budget']}") | |
| B, L = 2, CONFIG["seq_len"] | |
| input_ids = torch.randint(0, CONFIG["vocab_size"], (B, L)) | |
| logits, aux = model(input_ids) | |
| assert logits.shape == (B, L, CONFIG["vocab_size"]), \ | |
| f"logits shape {tuple(logits.shape)}" | |
| assert aux.shape == (), f"aux_loss must be scalar, got {tuple(aux.shape)}" | |
| print(f"\nforward OK -> logits {tuple(logits.shape)}, " | |
| f"aux_loss {aux.item():.6f}") | |
| # Loss + backward (sanity): CE over LM logits + aux. | |
| target = torch.randint(0, CONFIG["vocab_size"], (B, L)) | |
| loss = F.cross_entropy(logits.view(-1, CONFIG["vocab_size"]), | |
| target.view(-1)) | |
| total = loss + CONFIG["moe_aux_loss_weight"] * aux | |
| total.backward() | |
| print(f"loss+backward OK -> CE={loss.item():.4f} total={total.item():.4f}") | |
| # 5-token generate. | |
| prompt = torch.randint(0, CONFIG["vocab_size"], (B, 4)) | |
| out_ids = model.generate(prompt, max_length=5, temperature=0.7) | |
| assert out_ids.shape[0] == B | |
| assert out_ids.shape[1] >= 4 # at least the prompt | |
| assert out_ids.shape[1] <= 4 + 5 | |
| assert int(out_ids.max().item()) < CONFIG["vocab_size"] | |
| assert int(out_ids.min().item()) >= 0 | |
| print(f"generate OK -> {tuple(out_ids.shape)} " | |
| f"(prompt=4 + up to 5 new)") | |
| # Checkpoint round-trip in a temp path (no persistent ckpt/ dir β rule 8). | |
| with tempfile.TemporaryDirectory() as td: | |
| ck = os.path.join(td, "smoke.pt") | |
| model.save_checkpoint(ck) | |
| model2 = TinyMetatron.from_config() | |
| model2.load_checkpoint(ck) | |
| l2, a2 = model2(input_ids) | |
| assert l2.shape == logits.shape | |
| print("checkpoint round-trip OK") | |
| print("OK") |