Spaces:
Running
Running
Add mellum to KNOWN_BASES (JetBrains Mellum2-12B-A2.5B)
Browse files
scan.py
CHANGED
|
@@ -241,14 +241,25 @@ KNOWN_BASES = {
|
|
| 241 |
# Code-specialized tokenizer (49152 tokens)
|
| 242 |
},
|
| 243 |
"zaya": {
|
| 244 |
-
"name": "Zyphra ZAYA1 (
|
| 245 |
"vocab_size": 262272,
|
| 246 |
"model_type_patterns": ["zaya"],
|
| 247 |
-
#
|
| 248 |
-
#
|
| 249 |
-
#
|
| 250 |
-
#
|
| 251 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
},
|
| 253 |
}
|
| 254 |
|
|
|
|
| 241 |
# Code-specialized tokenizer (49152 tokens)
|
| 242 |
},
|
| 243 |
"zaya": {
|
| 244 |
+
"name": "Zyphra ZAYA1 (Global Attn + SWA + MoE)",
|
| 245 |
"vocab_size": 262272,
|
| 246 |
"model_type_patterns": ["zaya"],
|
| 247 |
+
# Global attention alternating with Sliding Window Attention (4K window) + MoE
|
| 248 |
+
# CCA (Cached Context Attention) is Zyphra's proprietary KV-efficient attn variant
|
| 249 |
+
# 8B: hidden=2048, 80 layers, 16 experts, 4B active
|
| 250 |
+
# 74B: hidden=4096, 120 layers, 24 experts, 4B active, GQA 16→2
|
| 251 |
+
# Depth (120L at 74B) enabled by SWA — halves KV cache vs all-global attention
|
| 252 |
+
# vocab ≈ Gemma tokenizer + 128 extra tokens; trained on AMD MI300x
|
| 253 |
+
# 74B-Preview is pre-RL reasoning base (no RLHF/instruct tuning)
|
| 254 |
+
},
|
| 255 |
+
"mellum": {
|
| 256 |
+
"name": "JetBrains Mellum (code-specialized MoE)",
|
| 257 |
+
"vocab_size": 98304,
|
| 258 |
+
"model_type_patterns": ["mellum"],
|
| 259 |
+
# Mellum2-12B-A2.5B: 28 layers all-MoE from layer 0, 64E/8A, GQA 32q/4kv (8:1),
|
| 260 |
+
# QK norm, custom MellumTopKRouter, 128K context, hidden=2304.
|
| 261 |
+
# vocab 98304 = 384×256 — code-optimized tokenizer (JetBrains IDE corpora).
|
| 262 |
+
# Mellum-4B (prior gen): 3.4B dense, 30 layers, hidden=3072, no MoE.
|
| 263 |
},
|
| 264 |
}
|
| 265 |
|