trohrbaugh commited on
Commit
5da97e4
·
verified ·
1 Parent(s): a1a2d2b

Add mellum to KNOWN_BASES (JetBrains Mellum2-12B-A2.5B)

Browse files
Files changed (1) hide show
  1. scan.py +17 -6
scan.py CHANGED
@@ -241,14 +241,25 @@ KNOWN_BASES = {
241
  # Code-specialized tokenizer (49152 tokens)
242
  },
243
  "zaya": {
244
- "name": "Zyphra ZAYA1 (deep hybrid SSM+Attention MoE)",
245
  "vocab_size": 262272,
246
  "model_type_patterns": ["zaya"],
247
- # Zyphra's hybrid SSM+attention MoE lineage (follows Zamba2 design philosophy)
248
- # 8B: hidden=2048, 80 layers, 16 experts, no explicit KV heads → SSM-dominant
249
- # 74B: hidden=4096, 120 layers, 24 experts, GQA 16→2 (extreme compression)
250
- # vocab ≈ Gemma tokenizer + 128 extra tokens (262144→262272)
251
- # Very deep-and-narrow: 120 layers at 74B vs Llama-3-70B's 80 layers
 
 
 
 
 
 
 
 
 
 
 
252
  },
253
  }
254
 
 
241
  # Code-specialized tokenizer (49152 tokens)
242
  },
243
  "zaya": {
244
+ "name": "Zyphra ZAYA1 (Global Attn + SWA + MoE)",
245
  "vocab_size": 262272,
246
  "model_type_patterns": ["zaya"],
247
+ # Global attention alternating with Sliding Window Attention (4K window) + MoE
248
+ # CCA (Cached Context Attention) is Zyphra's proprietary KV-efficient attn variant
249
+ # 8B: hidden=2048, 80 layers, 16 experts, 4B active
250
+ # 74B: hidden=4096, 120 layers, 24 experts, 4B active, GQA 16→2
251
+ # Depth (120L at 74B) enabled by SWA — halves KV cache vs all-global attention
252
+ # vocab ≈ Gemma tokenizer + 128 extra tokens; trained on AMD MI300x
253
+ # 74B-Preview is pre-RL reasoning base (no RLHF/instruct tuning)
254
+ },
255
+ "mellum": {
256
+ "name": "JetBrains Mellum (code-specialized MoE)",
257
+ "vocab_size": 98304,
258
+ "model_type_patterns": ["mellum"],
259
+ # Mellum2-12B-A2.5B: 28 layers all-MoE from layer 0, 64E/8A, GQA 32q/4kv (8:1),
260
+ # QK norm, custom MellumTopKRouter, 128K context, hidden=2304.
261
+ # vocab 98304 = 384×256 — code-optimized tokenizer (JetBrains IDE corpora).
262
+ # Mellum-4B (prior gen): 3.4B dense, 30 layers, hidden=3072, no MoE.
263
  },
264
  }
265