Instructions to use LocalWisdom/Penny-Alpha with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use LocalWisdom/Penny-Alpha with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="LocalWisdom/Penny-Alpha", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("LocalWisdom/Penny-Alpha", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use LocalWisdom/Penny-Alpha with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "LocalWisdom/Penny-Alpha" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "LocalWisdom/Penny-Alpha", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/LocalWisdom/Penny-Alpha
- SGLang
How to use LocalWisdom/Penny-Alpha with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "LocalWisdom/Penny-Alpha" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "LocalWisdom/Penny-Alpha", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "LocalWisdom/Penny-Alpha" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "LocalWisdom/Penny-Alpha", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use LocalWisdom/Penny-Alpha with Docker Model Runner:
docker model run hf.co/LocalWisdom/Penny-Alpha
Upload model.py with huggingface_hub
Browse files
model.py
ADDED
|
@@ -0,0 +1,689 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Penny (DeepSeek-V4.1-Flash port) for Hugging Face transformers, via trust_remote_code.
|
| 2 |
+
|
| 3 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 4 |
+
tok = AutoTokenizer.from_pretrained("LocalWisdom/Penny-Alpha")
|
| 5 |
+
model = AutoModelForCausalLM.from_pretrained("LocalWisdom/Penny-Alpha", trust_remote_code=True, dtype="float32")
|
| 6 |
+
ids = tok(tok.eos_token + "The history of the bicycle begins", add_special_tokens=False, return_tensors="pt")
|
| 7 |
+
out = model.generate(**ids, max_new_tokens=200, do_sample=True, temperature=0.8, top_k=50, use_cache=False)
|
| 8 |
+
print(tok.decode(out[0]))
|
| 9 |
+
|
| 10 |
+
Inference-only port of the training model (no KV cache: every step re-runs the whole sequence, which is
|
| 11 |
+
fine at this size). Weights use the official DeepSeek-V4.1 tensor names, so `model.safetensors` loads as is.
|
| 12 |
+
Penny is a base model (no chat template). Prompts must start with EOS (marks a fresh document); the tokenizer does not add it, so prepend tok.eos_token.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import math
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn.functional as F
|
| 21 |
+
from torch import nn
|
| 22 |
+
from transformers import AutoTokenizer, GenerationMixin, PretrainedConfig, PreTrainedModel
|
| 23 |
+
from transformers.modeling_outputs import CausalLMOutput
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# ---------------------------------------------------------------------------------------------
|
| 27 |
+
# config
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class DeepseekV41Config(PretrainedConfig):
|
| 31 |
+
"""Wraps the nested config.json: top-level HF fields + `text_config` (the architecture) + `penny`."""
|
| 32 |
+
|
| 33 |
+
model_type = "deepseek_v41"
|
| 34 |
+
|
| 35 |
+
def __init__(self, text_config=None, penny=None, **kwargs):
|
| 36 |
+
super().__init__(**kwargs)
|
| 37 |
+
# kept as `arch`, not `text_config`: transformers treats a `text_config` attribute as a sub-config object
|
| 38 |
+
self.arch = dict(text_config or {})
|
| 39 |
+
self.penny = dict(penny or {})
|
| 40 |
+
|
| 41 |
+
def to_dict(self):
|
| 42 |
+
d = super().to_dict()
|
| 43 |
+
d["text_config"] = d.pop("arch", self.arch)
|
| 44 |
+
return d
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass
|
| 48 |
+
class Args:
|
| 49 |
+
vocab_size: int
|
| 50 |
+
dim: int
|
| 51 |
+
moe_inter_dim: int
|
| 52 |
+
n_layers: int
|
| 53 |
+
n_heads: int
|
| 54 |
+
n_routed_experts: int
|
| 55 |
+
n_shared_experts: int
|
| 56 |
+
n_activated_experts: int
|
| 57 |
+
score_func: str
|
| 58 |
+
norm_topk_prob: bool
|
| 59 |
+
route_scale: float
|
| 60 |
+
swiglu_limit: float
|
| 61 |
+
q_lora_rank: int
|
| 62 |
+
head_dim: int
|
| 63 |
+
rope_head_dim: int
|
| 64 |
+
norm_eps: float
|
| 65 |
+
o_groups: int
|
| 66 |
+
o_lora_rank: int
|
| 67 |
+
window_size: int
|
| 68 |
+
compress_ratios: tuple
|
| 69 |
+
kv_source_layers: tuple
|
| 70 |
+
index_source_layers: tuple
|
| 71 |
+
compress_rope_theta: float
|
| 72 |
+
original_seq_len: int
|
| 73 |
+
rope_theta: float
|
| 74 |
+
rope_factor: float
|
| 75 |
+
beta_fast: int
|
| 76 |
+
beta_slow: int
|
| 77 |
+
index_n_heads: int
|
| 78 |
+
index_head_dim: int
|
| 79 |
+
index_topk: int
|
| 80 |
+
candidate_source_layer: int
|
| 81 |
+
candidate_topk_blocks: int
|
| 82 |
+
candidate_block_size: int
|
| 83 |
+
hc_mult: int
|
| 84 |
+
hc_sinkhorn_iters: int
|
| 85 |
+
hc_eps: float
|
| 86 |
+
engram_layer_ids: tuple
|
| 87 |
+
engram_num_embeddings: tuple
|
| 88 |
+
engram_max_ngram_size: int
|
| 89 |
+
engram_vocab_size: int
|
| 90 |
+
engram_n_heads: int
|
| 91 |
+
engram_head_dim: int
|
| 92 |
+
engram_pad_id: int
|
| 93 |
+
engram_compressed_vocab_size: int
|
| 94 |
+
max_seq_len: int
|
| 95 |
+
attn_chunk: int = 256
|
| 96 |
+
|
| 97 |
+
@classmethod
|
| 98 |
+
def from_config(cls, cfg: DeepseekV41Config) -> "Args":
|
| 99 |
+
t = cfg.arch
|
| 100 |
+
rs = t.get("rope_scaling") or {}
|
| 101 |
+
n = t["num_hidden_layers"]
|
| 102 |
+
return cls(
|
| 103 |
+
vocab_size=t["vocab_size"], dim=t["hidden_size"], moe_inter_dim=t["moe_intermediate_size"],
|
| 104 |
+
n_layers=n, n_heads=t["num_attention_heads"], n_routed_experts=t["n_routed_experts"],
|
| 105 |
+
n_shared_experts=t["n_shared_experts"], n_activated_experts=t["num_experts_per_tok"],
|
| 106 |
+
score_func=t["scoring_func"], norm_topk_prob=t["norm_topk_prob"],
|
| 107 |
+
route_scale=t["routed_scaling_factor"], swiglu_limit=t["swiglu_limit"],
|
| 108 |
+
q_lora_rank=t["q_lora_rank"], head_dim=t["head_dim"], rope_head_dim=t["qk_rope_head_dim"],
|
| 109 |
+
norm_eps=t["rms_norm_eps"], o_groups=t["o_groups"], o_lora_rank=t["o_lora_rank"],
|
| 110 |
+
window_size=t["sliding_window"], compress_ratios=tuple(t["compress_ratios"][:n]),
|
| 111 |
+
kv_source_layers=tuple(t["kv_source_layer_ids"]), index_source_layers=tuple(t["index_source_layer_ids"]),
|
| 112 |
+
compress_rope_theta=t["compress_rope_theta"],
|
| 113 |
+
original_seq_len=rs.get("original_max_position_embeddings", 0), rope_theta=t["rope_theta"],
|
| 114 |
+
rope_factor=rs.get("factor", 1), beta_fast=rs.get("beta_fast", 32), beta_slow=rs.get("beta_slow", 1),
|
| 115 |
+
index_n_heads=t["index_n_heads"], index_head_dim=t["index_head_dim"], index_topk=t["index_topk"],
|
| 116 |
+
candidate_source_layer=t["candidate_source_layer_id"], candidate_topk_blocks=t["candidate_topk_blocks"],
|
| 117 |
+
candidate_block_size=t["candidate_block_size"], hc_mult=t["hc_mult"],
|
| 118 |
+
hc_sinkhorn_iters=t["hc_sinkhorn_iters"], hc_eps=t["hc_eps"],
|
| 119 |
+
engram_layer_ids=tuple(t["engram_layer_ids"]), engram_num_embeddings=tuple(t["engram_num_embeddings"]),
|
| 120 |
+
engram_max_ngram_size=t["engram_max_ngram_size"], engram_vocab_size=t["engram_vocab_size"],
|
| 121 |
+
engram_n_heads=t["engram_n_heads"], engram_head_dim=t["engram_head_dim"],
|
| 122 |
+
engram_pad_id=t["engram_pad_token_id"], engram_compressed_vocab_size=t["engram_compressed_vocab_size"],
|
| 123 |
+
max_seq_len=cfg.penny.get("trained_seq_len", 8192),
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# ---------------------------------------------------------------------------------------------
|
| 128 |
+
# Engram n-gram hashing (hash ids match the DeepSeek reference exactly)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _is_prime(n: int) -> bool:
|
| 132 |
+
if n < 2:
|
| 133 |
+
return False
|
| 134 |
+
i = 2
|
| 135 |
+
while i * i <= n:
|
| 136 |
+
if n % i == 0:
|
| 137 |
+
return False
|
| 138 |
+
i += 1
|
| 139 |
+
return True
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _next_prime(start: int, seen: set) -> int:
|
| 143 |
+
c = start + 1
|
| 144 |
+
while not _is_prime(c) or c in seen:
|
| 145 |
+
c += 1
|
| 146 |
+
return c
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _compressed_token_map(tokenizer):
|
| 150 |
+
"""Collapse tokens that normalize alike (" The", "the", "THE") onto one compressed id."""
|
| 151 |
+
from tokenizers import Regex, normalizers
|
| 152 |
+
|
| 153 |
+
sentinel = "\ue000" # private-use char, as in the reference
|
| 154 |
+
normalizer = normalizers.Sequence([
|
| 155 |
+
normalizers.NFKC(), normalizers.NFD(), normalizers.StripAccents(), normalizers.Lowercase(),
|
| 156 |
+
normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), normalizers.Replace(Regex(r"^ $"), sentinel),
|
| 157 |
+
normalizers.Strip(), normalizers.Replace(sentinel, " "),
|
| 158 |
+
])
|
| 159 |
+
backend = tokenizer.backend_tokenizer
|
| 160 |
+
key_to_new, lookup = {}, [0] * len(tokenizer)
|
| 161 |
+
for token_id in range(len(tokenizer)):
|
| 162 |
+
text = backend.decode([token_id], skip_special_tokens=False)
|
| 163 |
+
if "\ufffd" in text:
|
| 164 |
+
key = backend.id_to_token(token_id)
|
| 165 |
+
else:
|
| 166 |
+
normalized = normalizer.normalize_str(text)
|
| 167 |
+
key = normalized if normalized else text
|
| 168 |
+
new_id = key_to_new.get(key)
|
| 169 |
+
if new_id is None:
|
| 170 |
+
new_id = len(key_to_new)
|
| 171 |
+
key_to_new[key] = new_id
|
| 172 |
+
lookup[token_id] = new_id
|
| 173 |
+
return lookup, len(key_to_new)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class NgramHash:
|
| 177 |
+
"""input_ids [B, L] -> hash ids [B, L, n_engram_layers, (max_ngram_size-1) * n_heads]. Built lazily
|
| 178 |
+
(needs the tokenizer) and not an nn.Module, so it never touches the weights."""
|
| 179 |
+
|
| 180 |
+
def __init__(self, a: Args, tokenizer):
|
| 181 |
+
layer_ids = tuple(a.engram_layer_ids)
|
| 182 |
+
self.max_n = a.engram_max_ngram_size
|
| 183 |
+
primes, seen = [], set()
|
| 184 |
+
for _ in layer_ids:
|
| 185 |
+
per_ngram = []
|
| 186 |
+
for _ in range(self.max_n - 1):
|
| 187 |
+
sizes, current = [], a.engram_vocab_size - 1
|
| 188 |
+
for _ in range(a.engram_n_heads):
|
| 189 |
+
current = _next_prime(current, seen)
|
| 190 |
+
seen.add(current)
|
| 191 |
+
sizes.append(current)
|
| 192 |
+
per_ngram.append(sizes)
|
| 193 |
+
primes.append(per_ngram)
|
| 194 |
+
token_map, vocab_size = _compressed_token_map(tokenizer)
|
| 195 |
+
assert vocab_size == a.engram_compressed_vocab_size, (vocab_size, a.engram_compressed_vocab_size)
|
| 196 |
+
self.pad_id = token_map[a.engram_pad_id]
|
| 197 |
+
flat = [[p for per_ngram in layer for p in per_ngram] for layer in primes]
|
| 198 |
+
offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat]
|
| 199 |
+
max_long = np.iinfo(np.int64).max
|
| 200 |
+
bound = max(1, (max_long // vocab_size) // 2)
|
| 201 |
+
mult = []
|
| 202 |
+
for layer_id in layer_ids:
|
| 203 |
+
g = np.random.default_rng(10007 * layer_id)
|
| 204 |
+
mult.append(torch.tensor(g.integers(low=0, high=bound, size=(self.max_n,), dtype=np.int64) * 2 + 1))
|
| 205 |
+
self.primes = torch.tensor(primes)
|
| 206 |
+
self.offsets = torch.tensor(np.array(offsets))
|
| 207 |
+
self.multipliers = torch.stack(mult)
|
| 208 |
+
self.token_map = torch.tensor(token_map)
|
| 209 |
+
|
| 210 |
+
@torch.no_grad()
|
| 211 |
+
def __call__(self, input_ids: torch.Tensor) -> torch.Tensor:
|
| 212 |
+
dev = input_ids.device
|
| 213 |
+
primes, offsets, multipliers, token_map = (
|
| 214 |
+
self.primes.to(dev), self.offsets.to(dev), self.multipliers.to(dev), self.token_map.to(dev))
|
| 215 |
+
compressed = token_map[input_ids]
|
| 216 |
+
tokens = [compressed]
|
| 217 |
+
for shift in range(1, self.max_n):
|
| 218 |
+
shifted = torch.full_like(compressed, self.pad_id)
|
| 219 |
+
shifted[:, shift:] = compressed[:, :-shift]
|
| 220 |
+
tokens.append(shifted)
|
| 221 |
+
tokens = torch.stack(tokens, dim=-1)
|
| 222 |
+
products = tokens.unsqueeze(2) * multipliers
|
| 223 |
+
rolling, hashes = products[..., 0], []
|
| 224 |
+
for i in range(1, self.max_n):
|
| 225 |
+
rolling = torch.bitwise_xor(rolling, products[..., i])
|
| 226 |
+
hashes.append(rolling.unsqueeze(-1) % primes[:, i - 1])
|
| 227 |
+
return torch.cat(hashes, dim=-1) + offsets
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
# ---------------------------------------------------------------------------------------------
|
| 231 |
+
# primitives
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
class RMSNorm(nn.Module):
|
| 235 |
+
def __init__(self, dim: int, eps: float):
|
| 236 |
+
super().__init__()
|
| 237 |
+
self.eps = eps
|
| 238 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 239 |
+
|
| 240 |
+
def forward(self, x):
|
| 241 |
+
dtype = x.dtype
|
| 242 |
+
x = x.float()
|
| 243 |
+
x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps)
|
| 244 |
+
return (self.weight.float() * x).to(dtype)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def precompute_freqs_cis(dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow, device):
|
| 248 |
+
freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
|
| 249 |
+
if original_seq_len > 0:
|
| 250 |
+
def corrected_dim(rotations):
|
| 251 |
+
return dim * math.log(original_seq_len / (rotations * 2 * math.pi)) / (2 * math.log(base))
|
| 252 |
+
|
| 253 |
+
low = max(math.floor(corrected_dim(beta_fast)), 0)
|
| 254 |
+
high = min(math.ceil(corrected_dim(beta_slow)), dim - 1)
|
| 255 |
+
ramp = ((torch.arange(dim // 2, dtype=torch.float32) - low) / max(high - low, 1e-3)).clamp(0, 1)
|
| 256 |
+
smooth = 1 - ramp
|
| 257 |
+
freqs = freqs / factor * (1 - smooth) + freqs * smooth
|
| 258 |
+
freqs = torch.outer(torch.arange(seqlen), freqs)
|
| 259 |
+
return torch.polar(torch.ones_like(freqs), freqs).to(device)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def rope(x, freqs_cis, rd, inverse=False):
|
| 263 |
+
"""Rotate the last `rd` dims of x (adjacent pairs as complex numbers). x: [b,s,d] or [b,s,h,d]."""
|
| 264 |
+
nope, pe = x[..., :-rd], x[..., -rd:]
|
| 265 |
+
z = torch.view_as_complex(pe.float().unflatten(-1, (-1, 2)).contiguous())
|
| 266 |
+
f = freqs_cis.conj() if inverse else freqs_cis
|
| 267 |
+
f = f.view(1, z.size(1), *([1] * (z.ndim - 3)), z.size(-1))
|
| 268 |
+
pe = torch.view_as_real(z * f).flatten(-2).to(x.dtype)
|
| 269 |
+
return torch.cat([nope, pe], dim=-1)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def idxs_to_mask(idxs, n):
|
| 273 |
+
"""[b,m,k] int indices (-1 = empty) -> [b,m,n] bool mask."""
|
| 274 |
+
mask = torch.zeros(*idxs.shape[:2], n + 1, dtype=torch.bool, device=idxs.device)
|
| 275 |
+
mask.scatter_(-1, torch.where(idxs < 0, n, idxs).long(), True)
|
| 276 |
+
return mask[..., :n]
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def chunked_sink_attention(q, kv, attn_sink, window, scale, comp_kv=None, comp_idxs=None, chunk=256):
|
| 280 |
+
"""Sliding-window attention over raw KV plus top-k compressed KV, with a learned sink logit.
|
| 281 |
+
q [b,L,h,d]; kv [b,L,d]; comp_kv [b,T,d]; comp_idxs [b,L,k] into comp_kv (-1 = empty)."""
|
| 282 |
+
L = q.size(1)
|
| 283 |
+
outs = []
|
| 284 |
+
for s0 in range(0, L, chunk):
|
| 285 |
+
s1 = min(s0 + chunk, L)
|
| 286 |
+
k0 = max(0, s0 - window + 1)
|
| 287 |
+
qpos = torch.arange(s0, s1, device=q.device).unsqueeze(1)
|
| 288 |
+
kpos = torch.arange(k0, s1, device=q.device)
|
| 289 |
+
wmask = (kpos <= qpos) & (kpos > qpos - window)
|
| 290 |
+
qc, kw = q[:, s0:s1], kv[:, k0:s1]
|
| 291 |
+
s = torch.einsum("bmhd,bnd->bhmn", qc, kw).float() * scale
|
| 292 |
+
s = s.masked_fill(~wmask, -torch.inf)
|
| 293 |
+
vals = kw
|
| 294 |
+
if comp_kv is not None:
|
| 295 |
+
cmask = idxs_to_mask(comp_idxs[:, s0:s1], comp_kv.size(1))
|
| 296 |
+
sc = torch.einsum("bmhd,bnd->bhmn", qc, comp_kv).float() * scale
|
| 297 |
+
s = torch.cat([s, sc.masked_fill(~cmask.unsqueeze(1), -torch.inf)], dim=-1)
|
| 298 |
+
vals = torch.cat([kw, comp_kv], dim=1)
|
| 299 |
+
sink = attn_sink.float().view(1, -1, 1, 1).expand(*s.shape[:3], 1)
|
| 300 |
+
lse = torch.logsumexp(torch.cat([s, sink], dim=-1), dim=-1)
|
| 301 |
+
p = torch.exp(s - lse.unsqueeze(-1))
|
| 302 |
+
outs.append(torch.einsum("bhmn,bnd->bmhd", p.to(vals.dtype), vals))
|
| 303 |
+
return torch.cat(outs, dim=1)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def hc_split_sinkhorn(mixes, hc_scale, hc_base, hc, iters, eps):
|
| 307 |
+
pre = torch.sigmoid(mixes[..., :hc] * hc_scale[0] + hc_base[:hc]) + eps
|
| 308 |
+
post = 2 * torch.sigmoid(mixes[..., hc : 2 * hc] * hc_scale[1] + hc_base[hc : 2 * hc])
|
| 309 |
+
comb = mixes[..., 2 * hc :] * hc_scale[2] + hc_base[2 * hc :]
|
| 310 |
+
comb = comb.unflatten(-1, (hc, hc)).softmax(-1) + eps
|
| 311 |
+
comb = comb / (comb.sum(-2, keepdim=True) + eps)
|
| 312 |
+
for _ in range(iters - 1):
|
| 313 |
+
comb = comb / (comb.sum(-1, keepdim=True) + eps)
|
| 314 |
+
comb = comb / (comb.sum(-2, keepdim=True) + eps)
|
| 315 |
+
return pre, post, comb
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
# ---------------------------------------------------------------------------------------------
|
| 319 |
+
# attention
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
class Shared:
|
| 323 |
+
"""Per-forward hand-off between source layers and the layers that reuse their results."""
|
| 324 |
+
|
| 325 |
+
def __init__(self):
|
| 326 |
+
self.compress_kv = None
|
| 327 |
+
self.index_k = None
|
| 328 |
+
self.topk_idxs = None
|
| 329 |
+
self.candidates = None
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
class Compressor(nn.Module):
|
| 333 |
+
"""Pools `ratio` consecutive tokens into one KV latent with a learned softmax gate (pre-RoPE)."""
|
| 334 |
+
|
| 335 |
+
def __init__(self, a: Args, layer_id: int):
|
| 336 |
+
super().__init__()
|
| 337 |
+
self.ratio = a.compress_ratios[layer_id]
|
| 338 |
+
self.norm = RMSNorm(a.head_dim, a.norm_eps)
|
| 339 |
+
self.wkv = nn.Linear(a.dim, a.head_dim, bias=False)
|
| 340 |
+
self.wgate = nn.Linear(a.dim, a.head_dim, bias=False) if self.ratio > 1 else None
|
| 341 |
+
|
| 342 |
+
def forward(self, x):
|
| 343 |
+
if self.ratio == 1:
|
| 344 |
+
return self.norm(self.wkv(x))
|
| 345 |
+
cutoff = x.size(1) - x.size(1) % self.ratio # trailing partial group is not visible yet
|
| 346 |
+
xf = x[:, :cutoff].float() # the reference pools in fp32
|
| 347 |
+
kv = F.linear(xf, self.wkv.weight.float()).unflatten(1, (-1, self.ratio))
|
| 348 |
+
score = F.linear(xf, self.wgate.weight.float()).unflatten(1, (-1, self.ratio))
|
| 349 |
+
kv = (kv * score.softmax(dim=2)).sum(dim=2)
|
| 350 |
+
return self.norm(kv.to(x.dtype))
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def select_candidate_blocks(logits, compress_lens, topk_blocks, block_size):
|
| 354 |
+
width = logits.size(-1)
|
| 355 |
+
scores = F.pad(logits, (0, -width % block_size), value=-torch.inf)
|
| 356 |
+
scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1)
|
| 357 |
+
num_blocks = scores.size(-1)
|
| 358 |
+
last = (compress_lens - 1) // block_size
|
| 359 |
+
scores = scores.masked_fill(torch.arange(num_blocks, device=logits.device) == last, torch.inf)
|
| 360 |
+
top = scores.topk(min(topk_blocks, num_blocks), dim=-1)
|
| 361 |
+
keep = torch.zeros_like(scores, dtype=torch.bool).scatter_(-1, top.indices, top.values > -torch.inf)
|
| 362 |
+
return keep.repeat_interleave(block_size, dim=-1)[..., :width]
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
class Indexer(nn.Module):
|
| 366 |
+
"""Scores compressed positions and keeps the `index_topk` best per query."""
|
| 367 |
+
|
| 368 |
+
def __init__(self, a: Args, layer_id: int):
|
| 369 |
+
super().__init__()
|
| 370 |
+
self.owns_k = layer_id in a.kv_source_layers
|
| 371 |
+
self.ratio = a.compress_ratios[layer_id]
|
| 372 |
+
self.is_candidate_source = layer_id == a.candidate_source_layer
|
| 373 |
+
self.uses_candidates = 0 <= a.candidate_source_layer < layer_id
|
| 374 |
+
self.candidate_topk_blocks = a.candidate_topk_blocks
|
| 375 |
+
self.candidate_block_size = a.candidate_block_size
|
| 376 |
+
self.n_heads = a.index_n_heads
|
| 377 |
+
self.index_head_dim = a.index_head_dim
|
| 378 |
+
self.rd = a.rope_head_dim
|
| 379 |
+
self.index_topk = a.index_topk
|
| 380 |
+
self.softmax_scale = self.index_head_dim**-0.5
|
| 381 |
+
self.wq_b = nn.Linear(a.q_lora_rank, self.n_heads * self.index_head_dim, bias=False)
|
| 382 |
+
self.weights_proj = nn.Linear(a.dim, self.n_heads, bias=False)
|
| 383 |
+
if self.owns_k:
|
| 384 |
+
self.wk = nn.Linear(a.head_dim, self.index_head_dim, bias=False)
|
| 385 |
+
self.k_norm = RMSNorm(self.index_head_dim, a.norm_eps)
|
| 386 |
+
|
| 387 |
+
@torch.no_grad()
|
| 388 |
+
def forward(self, x, qr, latent, freqs_cis, shared: Shared, chunk: int):
|
| 389 |
+
"""Returns selected compressed indices [b,s,k] (-1 = empty)."""
|
| 390 |
+
seqlen, ratio = x.size(1), self.ratio
|
| 391 |
+
if self.owns_k:
|
| 392 |
+
k = self.k_norm(self.wk(latent))
|
| 393 |
+
shared.index_k = rope(k, freqs_cis[: seqlen - seqlen % ratio : ratio], self.rd)
|
| 394 |
+
q = rope(self.wq_b(qr).unflatten(-1, (self.n_heads, self.index_head_dim)), freqs_cis[:seqlen], self.rd)
|
| 395 |
+
index_k = shared.index_k
|
| 396 |
+
weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads**-0.5)
|
| 397 |
+
T = index_k.size(1)
|
| 398 |
+
topk = min(self.index_topk, T)
|
| 399 |
+
idx_chunks, cand_chunks = [], []
|
| 400 |
+
for s0 in range(0, seqlen, chunk):
|
| 401 |
+
s1 = min(s0 + chunk, seqlen)
|
| 402 |
+
score = torch.einsum("bshd,btd->bsht", q[:, s0:s1], index_k)
|
| 403 |
+
score = (score.relu() * weights[:, s0:s1].unsqueeze(-1)).sum(dim=2).float()
|
| 404 |
+
lens = (torch.arange(s0 + 1, s1 + 1, device=x.device) // ratio).unsqueeze(-1)
|
| 405 |
+
score = score.masked_fill(torch.arange(T, device=x.device) >= lens, -torch.inf)
|
| 406 |
+
if self.is_candidate_source:
|
| 407 |
+
cand_chunks.append(select_candidate_blocks(
|
| 408 |
+
score, lens, self.candidate_topk_blocks, self.candidate_block_size))
|
| 409 |
+
elif self.uses_candidates and shared.candidates is not None:
|
| 410 |
+
score = score.masked_fill(~shared.candidates[:, s0:s1], -torch.inf)
|
| 411 |
+
idxs = score.topk(topk, dim=-1, sorted=False).indices
|
| 412 |
+
idx_chunks.append(torch.where(idxs < lens, idxs, -1))
|
| 413 |
+
if self.is_candidate_source:
|
| 414 |
+
shared.candidates = torch.cat(cand_chunks, dim=1)
|
| 415 |
+
return torch.cat(idx_chunks, dim=1)
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
class Attention(nn.Module):
|
| 419 |
+
"""Sliding window over raw KV, plus (compress_ratio > 0) top-k compressed KV further back.
|
| 420 |
+
Low-rank Q; grouped low-rank output projection; shared K=V latent (MQA)."""
|
| 421 |
+
|
| 422 |
+
def __init__(self, layer_id: int, a: Args):
|
| 423 |
+
super().__init__()
|
| 424 |
+
self.n_heads, self.head_dim, self.rd = a.n_heads, a.head_dim, a.rope_head_dim
|
| 425 |
+
self.n_groups, self.o_lora_rank, self.window = a.o_groups, a.o_lora_rank, a.window_size
|
| 426 |
+
self.ratio = a.compress_ratios[layer_id]
|
| 427 |
+
self.softmax_scale = self.head_dim**-0.5
|
| 428 |
+
self.attn_chunk = a.attn_chunk
|
| 429 |
+
self.attn_sink = nn.Parameter(torch.zeros(self.n_heads))
|
| 430 |
+
self.wq_a = nn.Linear(a.dim, a.q_lora_rank, bias=False)
|
| 431 |
+
self.q_norm = RMSNorm(a.q_lora_rank, a.norm_eps)
|
| 432 |
+
self.wq_b = nn.Linear(a.q_lora_rank, self.n_heads * self.head_dim, bias=False)
|
| 433 |
+
self.wkv = nn.Linear(a.dim, self.head_dim, bias=False)
|
| 434 |
+
self.kv_norm = RMSNorm(self.head_dim, a.norm_eps)
|
| 435 |
+
self.wo_a = nn.Linear(self.n_heads * self.head_dim // self.n_groups, self.n_groups * self.o_lora_rank, bias=False)
|
| 436 |
+
self.wo_b = nn.Linear(self.n_groups * self.o_lora_rank, a.dim, bias=False)
|
| 437 |
+
self.is_kv_source = layer_id in a.kv_source_layers
|
| 438 |
+
self.is_index_source = layer_id in a.index_source_layers
|
| 439 |
+
self.compressor = Compressor(a, layer_id) if self.is_kv_source else None
|
| 440 |
+
self.indexer = Indexer(a, layer_id) if self.is_index_source else None
|
| 441 |
+
# RoPE table: built lazily on the right device (never part of the checkpoint)
|
| 442 |
+
if self.ratio:
|
| 443 |
+
self._rope = (a.original_seq_len, a.compress_rope_theta)
|
| 444 |
+
else:
|
| 445 |
+
self._rope = (0, a.rope_theta) # pure sliding window: no YaRN, base theta
|
| 446 |
+
self._rope_args = (a.rope_factor, a.beta_fast, a.beta_slow, a.max_seq_len)
|
| 447 |
+
self._fc = None
|
| 448 |
+
|
| 449 |
+
def freqs(self, seqlen, device):
|
| 450 |
+
if self._fc is None or self._fc.size(0) < seqlen or self._fc.device != device:
|
| 451 |
+
factor, bf, bs, max_len = self._rope_args
|
| 452 |
+
self._fc = precompute_freqs_cis(self.rd, max(seqlen, max_len), self._rope[0], self._rope[1],
|
| 453 |
+
factor, bf, bs, device)
|
| 454 |
+
return self._fc[:seqlen]
|
| 455 |
+
|
| 456 |
+
def forward(self, x, shared: Shared):
|
| 457 |
+
bsz, seqlen, _ = x.shape
|
| 458 |
+
fc = self.freqs(seqlen, x.device)
|
| 459 |
+
qr = self.q_norm(self.wq_a(x))
|
| 460 |
+
q = rope(self.wq_b(qr).unflatten(-1, (self.n_heads, self.head_dim)), fc, self.rd)
|
| 461 |
+
kv = rope(self.kv_norm(self.wkv(x)), fc, self.rd)
|
| 462 |
+
|
| 463 |
+
comp_kv, cidx = None, None
|
| 464 |
+
if self.ratio:
|
| 465 |
+
latent = self.compressor(x) if self.is_kv_source else None
|
| 466 |
+
if self.is_index_source:
|
| 467 |
+
shared.topk_idxs = self.indexer(x, qr, latent, fc, shared, chunk=self.attn_chunk)
|
| 468 |
+
cidx = shared.topk_idxs
|
| 469 |
+
if latent is not None: # group j sits at the position of its first token, j * ratio
|
| 470 |
+
shared.compress_kv = rope(latent, fc[: seqlen - seqlen % self.ratio : self.ratio], self.rd)
|
| 471 |
+
comp_kv = shared.compress_kv
|
| 472 |
+
|
| 473 |
+
o = chunked_sink_attention(q, kv, self.attn_sink, self.window, self.softmax_scale, comp_kv, cidx,
|
| 474 |
+
chunk=self.attn_chunk)
|
| 475 |
+
o = rope(o, fc, self.rd, inverse=True) # remove the query rotation from the output
|
| 476 |
+
o = o.reshape(bsz, seqlen, self.n_groups, -1)
|
| 477 |
+
wo_a = self.wo_a.weight.view(self.n_groups, self.o_lora_rank, -1)
|
| 478 |
+
o = torch.einsum("bsgd,grd->bsgr", o, wo_a.to(o.dtype))
|
| 479 |
+
return self.wo_b(o.flatten(2))
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
# ---------------------------------------------------------------------------------------------
|
| 483 |
+
# MoE
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
class Gate(nn.Module):
|
| 487 |
+
"""sqrtsoftplus scores; a bias picks experts, weights come from the unbiased scores."""
|
| 488 |
+
|
| 489 |
+
def __init__(self, a: Args):
|
| 490 |
+
super().__init__()
|
| 491 |
+
self.topk, self.score_func = a.n_activated_experts, a.score_func
|
| 492 |
+
self.norm_topk_prob, self.route_scale = a.norm_topk_prob, a.route_scale
|
| 493 |
+
self.weight = nn.Parameter(torch.empty(a.n_routed_experts, a.dim))
|
| 494 |
+
self.register_buffer("bias", torch.zeros(a.n_routed_experts))
|
| 495 |
+
|
| 496 |
+
def forward(self, x):
|
| 497 |
+
scores = F.linear(x.float(), self.weight.float())
|
| 498 |
+
if self.score_func == "softmax":
|
| 499 |
+
scores = scores.softmax(dim=-1)
|
| 500 |
+
elif self.score_func == "sigmoid":
|
| 501 |
+
scores = scores.sigmoid()
|
| 502 |
+
else:
|
| 503 |
+
scores = F.softplus(scores).sqrt()
|
| 504 |
+
indices = (scores + self.bias.float()).topk(self.topk, dim=-1)[1]
|
| 505 |
+
weights = scores.gather(1, indices)
|
| 506 |
+
if self.norm_topk_prob and self.topk > 1:
|
| 507 |
+
weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20)
|
| 508 |
+
return weights * self.route_scale, indices
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
class Expert(nn.Module):
|
| 512 |
+
def __init__(self, dim, inter_dim, swiglu_limit):
|
| 513 |
+
super().__init__()
|
| 514 |
+
self.w1 = nn.Linear(dim, inter_dim, bias=False)
|
| 515 |
+
self.w2 = nn.Linear(inter_dim, dim, bias=False)
|
| 516 |
+
self.w3 = nn.Linear(dim, inter_dim, bias=False)
|
| 517 |
+
self.swiglu_limit = swiglu_limit
|
| 518 |
+
|
| 519 |
+
def forward(self, x, weights=None):
|
| 520 |
+
dtype = x.dtype
|
| 521 |
+
gate, up = self.w1(x).float(), self.w3(x).float()
|
| 522 |
+
if self.swiglu_limit > 0:
|
| 523 |
+
up = up.clamp(-self.swiglu_limit, self.swiglu_limit)
|
| 524 |
+
gate = gate.clamp(max=self.swiglu_limit)
|
| 525 |
+
h = F.silu(gate) * up
|
| 526 |
+
if weights is not None:
|
| 527 |
+
h = weights * h
|
| 528 |
+
return self.w2(h.to(dtype))
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
class MoE(nn.Module):
|
| 532 |
+
def __init__(self, a: Args):
|
| 533 |
+
super().__init__()
|
| 534 |
+
self.dim = a.dim
|
| 535 |
+
self.gate = Gate(a)
|
| 536 |
+
self.experts = nn.ModuleList([Expert(a.dim, a.moe_inter_dim, a.swiglu_limit) for _ in range(a.n_routed_experts)])
|
| 537 |
+
assert a.n_shared_experts == 1
|
| 538 |
+
self.shared_experts = Expert(a.dim, a.moe_inter_dim, a.swiglu_limit)
|
| 539 |
+
|
| 540 |
+
def forward(self, x):
|
| 541 |
+
shape = x.shape
|
| 542 |
+
x = x.reshape(-1, self.dim)
|
| 543 |
+
weights, indices = self.gate(x)
|
| 544 |
+
y = torch.zeros_like(x, dtype=torch.float32)
|
| 545 |
+
counts = torch.bincount(indices.flatten(), minlength=len(self.experts)).tolist()
|
| 546 |
+
for i, expert in enumerate(self.experts):
|
| 547 |
+
if counts[i] == 0:
|
| 548 |
+
continue
|
| 549 |
+
idx, top = torch.where(indices == i)
|
| 550 |
+
y = y.index_add(0, idx, expert(x[idx], weights[idx, top, None]).float())
|
| 551 |
+
y = y + self.shared_experts(x).float()
|
| 552 |
+
return y.to(x.dtype).view(shape)
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
# ---------------------------------------------------------------------------------------------
|
| 556 |
+
# Engram, blocks, model
|
| 557 |
+
|
| 558 |
+
|
| 559 |
+
class Engram(nn.Module):
|
| 560 |
+
"""Gated n-gram lookup written into each hc copy of the residual stream."""
|
| 561 |
+
|
| 562 |
+
def __init__(self, a: Args, layer_id: int):
|
| 563 |
+
super().__init__()
|
| 564 |
+
self.layer_hash_index = tuple(a.engram_layer_ids).index(layer_id)
|
| 565 |
+
self.dim, self.hc_mult, self.eps = a.dim, a.hc_mult, a.norm_eps
|
| 566 |
+
self.embed = nn.Embedding(a.engram_num_embeddings[self.layer_hash_index], a.engram_head_dim)
|
| 567 |
+
n_hash_cols = (a.engram_max_ngram_size - 1) * a.engram_n_heads
|
| 568 |
+
self.wkv = nn.Linear(n_hash_cols * a.engram_head_dim, a.dim * (a.hc_mult + 1), bias=False)
|
| 569 |
+
self.q_weight = nn.Parameter(torch.ones(a.hc_mult, a.dim))
|
| 570 |
+
self.k_weight = nn.Parameter(torch.ones(a.hc_mult, a.dim))
|
| 571 |
+
|
| 572 |
+
def forward(self, x, hash_ids):
|
| 573 |
+
kv = self.wkv(self.embed(hash_ids).flatten(-2))
|
| 574 |
+
key, value = kv.split([self.hc_mult * self.dim, self.dim], dim=-1)
|
| 575 |
+
key = key.float().unflatten(-1, (self.hc_mult, self.dim))
|
| 576 |
+
h = x.float()
|
| 577 |
+
rstd = torch.rsqrt(h.square().mean(-1) + self.eps) * torch.rsqrt(key.square().mean(-1) + self.eps)
|
| 578 |
+
dot = (h * (self.q_weight.float() * self.k_weight.float()) * key).sum(-1) * rstd * self.dim**-0.5
|
| 579 |
+
gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(1e-6).sqrt(), dot))
|
| 580 |
+
return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype)
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
class Block(nn.Module):
|
| 584 |
+
"""Hyper-Connections block: the residual stream is carried as hc_mult copies."""
|
| 585 |
+
|
| 586 |
+
def __init__(self, layer_id: int, a: Args):
|
| 587 |
+
super().__init__()
|
| 588 |
+
self.norm_eps = a.norm_eps
|
| 589 |
+
self.attn = Attention(layer_id, a)
|
| 590 |
+
self.ffn = MoE(a)
|
| 591 |
+
self.engram = Engram(a, layer_id) if layer_id in a.engram_layer_ids else None
|
| 592 |
+
self.attn_norm = RMSNorm(a.dim, a.norm_eps)
|
| 593 |
+
self.ffn_norm = RMSNorm(a.dim, a.norm_eps)
|
| 594 |
+
self.hc, self.hc_iters, self.hc_eps = a.hc_mult, a.hc_sinkhorn_iters, a.hc_eps
|
| 595 |
+
mix_hc = (2 + self.hc) * self.hc
|
| 596 |
+
self.hc_attn_fn = nn.Parameter(torch.empty(mix_hc, self.hc * a.dim))
|
| 597 |
+
self.hc_ffn_fn = nn.Parameter(torch.empty(mix_hc, self.hc * a.dim))
|
| 598 |
+
self.hc_attn_base = nn.Parameter(torch.empty(mix_hc))
|
| 599 |
+
self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc))
|
| 600 |
+
self.hc_attn_scale = nn.Parameter(torch.ones(3))
|
| 601 |
+
self.hc_ffn_scale = nn.Parameter(torch.ones(3))
|
| 602 |
+
|
| 603 |
+
def hc_mixes(self, x, fn, scale, base):
|
| 604 |
+
x = x.flatten(2).float()
|
| 605 |
+
mixes = F.linear(x, fn.float()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.norm_eps)
|
| 606 |
+
return hc_split_sinkhorn(mixes, scale.float(), base.float(), self.hc, self.hc_iters, self.hc_eps)
|
| 607 |
+
|
| 608 |
+
@staticmethod
|
| 609 |
+
def hc_pre(x, pre_mix):
|
| 610 |
+
return torch.sum(pre_mix.unsqueeze(-1) * x.float(), dim=2).to(x.dtype)
|
| 611 |
+
|
| 612 |
+
@staticmethod
|
| 613 |
+
def hc_post(x, residual, post, comb):
|
| 614 |
+
y = post.unsqueeze(-1) * x.unsqueeze(-2) + torch.sum(comb.unsqueeze(-1) * residual.unsqueeze(-2), dim=2)
|
| 615 |
+
return y.to(residual.dtype)
|
| 616 |
+
|
| 617 |
+
def forward(self, x, pre_mix, shared: Shared):
|
| 618 |
+
residual = x
|
| 619 |
+
attn_pre, attn_post, attn_comb = self.hc_mixes(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
|
| 620 |
+
h = self.attn(self.attn_norm(self.hc_pre(x, pre_mix)), shared)
|
| 621 |
+
x = self.hc_post(h, residual, attn_post, attn_comb)
|
| 622 |
+
residual = x
|
| 623 |
+
ffn_pre, ffn_post, ffn_comb = self.hc_mixes(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)
|
| 624 |
+
h = self.ffn(self.ffn_norm(self.hc_pre(x, attn_pre)))
|
| 625 |
+
x = self.hc_post(h, residual, ffn_post, ffn_comb)
|
| 626 |
+
return x, ffn_pre
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
class DeepseekV41PreTrainedModel(PreTrainedModel):
|
| 630 |
+
config_class = DeepseekV41Config
|
| 631 |
+
base_model_prefix = ""
|
| 632 |
+
supports_gradient_checkpointing = False
|
| 633 |
+
|
| 634 |
+
def _init_weights(self, module):
|
| 635 |
+
pass # every parameter comes from model.safetensors
|
| 636 |
+
|
| 637 |
+
|
| 638 |
+
class DeepseekV41ForCausalLM(DeepseekV41PreTrainedModel, GenerationMixin):
|
| 639 |
+
"""Submodule names (embed, layers, norm, head) match the checkpoint's tensor names directly."""
|
| 640 |
+
|
| 641 |
+
def __init__(self, config: DeepseekV41Config):
|
| 642 |
+
super().__init__(config)
|
| 643 |
+
a = Args.from_config(config)
|
| 644 |
+
self.args = a
|
| 645 |
+
self.embed = nn.Embedding(a.vocab_size, a.dim)
|
| 646 |
+
self.layers = nn.ModuleList([Block(i, a) for i in range(a.n_layers)])
|
| 647 |
+
self.norm = RMSNorm(a.dim, a.norm_eps)
|
| 648 |
+
self.head = nn.Linear(a.dim, a.vocab_size, bias=False)
|
| 649 |
+
self.hc = a.hc_mult
|
| 650 |
+
self._engram_hash = None
|
| 651 |
+
self.post_init()
|
| 652 |
+
|
| 653 |
+
def get_input_embeddings(self):
|
| 654 |
+
return self.embed
|
| 655 |
+
|
| 656 |
+
def set_input_embeddings(self, value):
|
| 657 |
+
self.embed = value
|
| 658 |
+
|
| 659 |
+
def get_output_embeddings(self):
|
| 660 |
+
return self.head
|
| 661 |
+
|
| 662 |
+
def set_output_embeddings(self, value):
|
| 663 |
+
self.head = value
|
| 664 |
+
|
| 665 |
+
def _engram(self):
|
| 666 |
+
if self._engram_hash is None: # needs the tokenizer shipped in the same repo / folder
|
| 667 |
+
src = getattr(self.config, "_name_or_path", None) or self.config.name_or_path
|
| 668 |
+
self._engram_hash = NgramHash(self.args, AutoTokenizer.from_pretrained(src))
|
| 669 |
+
return self._engram_hash
|
| 670 |
+
|
| 671 |
+
def forward(self, input_ids=None, attention_mask=None, labels=None, logits_to_keep=0, **kwargs):
|
| 672 |
+
"""No KV cache; attention_mask is ignored (use one unpadded sequence per row, i.e. batch of equal
|
| 673 |
+
lengths or batch size 1). logits_to_keep=N returns logits for the last N positions only."""
|
| 674 |
+
bsz, seqlen = input_ids.shape
|
| 675 |
+
hashes = self._engram()(input_ids)
|
| 676 |
+
h = self.embed(input_ids).unsqueeze(2).repeat(1, 1, self.hc, 1)
|
| 677 |
+
pre_mix = torch.zeros(bsz, seqlen, self.hc, device=h.device)
|
| 678 |
+
pre_mix[..., 0] = 1.0
|
| 679 |
+
shared = Shared()
|
| 680 |
+
for layer in self.layers:
|
| 681 |
+
if layer.engram is not None:
|
| 682 |
+
h = layer.engram(h, hashes[:, :, layer.engram.layer_hash_index])
|
| 683 |
+
h, pre_mix = layer(h, pre_mix, shared)
|
| 684 |
+
h = self.norm(Block.hc_pre(h, pre_mix))
|
| 685 |
+
logits = self.head(h[:, -logits_to_keep:] if logits_to_keep else h)
|
| 686 |
+
loss = None
|
| 687 |
+
if labels is not None:
|
| 688 |
+
loss = F.cross_entropy(logits[:, :-1].float().flatten(0, 1), labels[:, 1:].flatten(), ignore_index=-100)
|
| 689 |
+
return CausalLMOutput(loss=loss, logits=logits)
|