Upload int4 quantized MD3 weights

README.md

@@ -0,0 +1,31 @@
+---
+license: apache-2.0
+tags:
+- vision
+- moondream
+- mlx
+- int4
+---
+
+# MD3 Preview - Int4 Quantized (MLX)
+
+Pre-quantized version of Moondream 3 Preview for MLX inference.
+
+## Quantization Details
+
+- **MoE Experts**: int4 affine quantization (bits=4, group_size=64)
+- **Other weights**: bf16 (unchanged)
+- **Memory savings**: ~60% reduction in MoE weight memory
+
+## Usage
+
+This model is designed for use with the moondream-station MLX backend.
+
+```python
+# In moondream-station, use with:
+moondream-station serve --backend mlx
+```
+
+## Source
+
+Quantized from [moondream/moondream3-preview](https://huggingface.co/moondream/moondream3-preview)

config.json

@@ -0,0 +1,59 @@
+{
+  "architectures": [
+    "HfMoondream"
+  ],
+  "auto_map": {
+    "AutoConfig": "hf_moondream.HfConfig",
+    "AutoModelForCausalLM": "hf_moondream.HfMoondream"
+  },
+  "config": {
+    "skills": [
+      "query",
+      "caption",
+      "detect",
+      "point"
+    ]
+  },
+  "model_type": "moondream3",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.1",
+  "text": {
+    "dim": 2048,
+    "ff_dim": 8192,
+    "n_layers": 24,
+    "vocab_size": 51200,
+    "max_context": 4096,
+    "n_heads": 32,
+    "n_kv_heads": 32,
+    "prefix_attn": 730,
+    "group_size": null,
+    "moe": {
+      "num_experts": 64,
+      "start_layer": 4,
+      "experts_per_token": 8,
+      "expert_inner_dim": 1024
+    }
+  },
+  "vision": {
+    "enc_dim": 1152,
+    "enc_patch_size": 14,
+    "enc_n_layers": 27,
+    "enc_ff_dim": 4304,
+    "enc_n_heads": 16,
+    "proj_out_dim": 2048,
+    "crop_size": 378,
+    "in_channels": 3,
+    "max_crops": 12,
+    "overlap_margin": 4,
+    "proj_inner_dim": 8192
+  },
+  "region": {
+    "dim": 2048,
+    "coord_feat_dim": 256,
+    "coord_out_dim": 1024,
+    "size_feat_dim": 512,
+    "size_out_dim": 2048,
+    "group_size": null
+  },
+  "dtype": "bfloat16"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd9197e2c82d0e1f46ef732d6ebe361e7587dd96936b7f5eb627e79eb6ec979f
+size 6961043417

model.safetensors.index.json

@@ -0,0 +1,747 @@
+{
+  "metadata": {
+    "total_size": 6960966624
+  },
+  "weight_map": {
+    "text.blocks.19.ln.weight": "model.safetensors",
+    "text.blocks.19.ln.bias": "model.safetensors",
+    "text.blocks.19.attn.tau_alpha": "model.safetensors",
+    "text.blocks.19.attn.qkv.weight": "model.safetensors",
+    "text.blocks.18.mlp.fc2_q": "model.safetensors",
+    "text.blocks.18.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.18.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.18.mlp.fc1_q": "model.safetensors",
+    "text.blocks.18.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.18.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.18.ln.weight": "model.safetensors",
+    "text.blocks.18.attn.qkv.weight": "model.safetensors",
+    "text.blocks.18.attn.proj.weight": "model.safetensors",
+    "text.blocks.18.attn.proj.bias": "model.safetensors",
+    "text.blocks.17.mlp.router.weight": "model.safetensors",
+    "text.blocks.18.mlp.router.bias": "model.safetensors",
+    "text.blocks.17.mlp.fc2_q": "model.safetensors",
+    "text.blocks.17.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.17.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.19.attn.tau_wq": "model.safetensors",
+    "text.blocks.17.mlp.fc1_q": "model.safetensors",
+    "text.blocks.17.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.17.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.19.mlp.router.bias": "model.safetensors",
+    "text.blocks.17.ln.weight": "model.safetensors",
+    "text.blocks.17.ln.bias": "model.safetensors",
+    "text.blocks.14.ln.bias": "model.safetensors",
+    "text.blocks.17.attn.tau_wq": "model.safetensors",
+    "text.blocks.17.attn.proj.weight": "model.safetensors",
+    "text.blocks.15.attn.tau_alpha": "model.safetensors",
+    "text.blocks.17.attn.proj.bias": "model.safetensors",
+    "text.blocks.17.attn.qkv.weight": "model.safetensors",
+    "text.blocks.16.mlp.router.weight": "model.safetensors",
+    "text.blocks.16.ln.weight": "model.safetensors",
+    "text.blocks.16.attn.proj.bias": "model.safetensors",
+    "text.blocks.16.attn.tau_wv": "model.safetensors",
+    "text.blocks.16.attn.qkv.weight": "model.safetensors",
+    "text.blocks.15.attn.qkv.bias": "model.safetensors",
+    "text.blocks.15.mlp.router.bias": "model.safetensors",
+    "text.blocks.17.attn.qkv.bias": "model.safetensors",
+    "text.blocks.15.attn.tau_wq": "model.safetensors",
+    "text.blocks.15.ln.weight": "model.safetensors",
+    "text.blocks.14.ln.weight": "model.safetensors",
+    "text.blocks.15.mlp.fc1_q": "model.safetensors",
+    "text.blocks.15.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.15.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.15.attn.tau_wv": "model.safetensors",
+    "text.blocks.19.mlp.router.weight": "model.safetensors",
+    "text.blocks.17.attn.tau_wv": "model.safetensors",
+    "text.blocks.13.mlp.fc2_q": "model.safetensors",
+    "text.blocks.13.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.13.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.15.mlp.router.weight": "model.safetensors",
+    "text.blocks.14.mlp.router.weight": "model.safetensors",
+    "text.blocks.17.attn.tau_alpha": "model.safetensors",
+    "text.blocks.19.attn.qkv.bias": "model.safetensors",
+    "text.blocks.14.attn.tau_wv": "model.safetensors",
+    "text.blocks.16.attn.proj.weight": "model.safetensors",
+    "text.blocks.14.mlp.fc1_q": "model.safetensors",
+    "text.blocks.14.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.14.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.16.mlp.router.bias": "model.safetensors",
+    "text.blocks.16.mlp.fc2_q": "model.safetensors",
+    "text.blocks.16.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.16.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.15.mlp.fc2_q": "model.safetensors",
+    "text.blocks.15.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.15.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.18.attn.qkv.bias": "model.safetensors",
+    "text.blocks.14.attn.qkv.weight": "model.safetensors",
+    "text.blocks.18.attn.tau_wq": "model.safetensors",
+    "text.blocks.18.attn.tau_alpha": "model.safetensors",
+    "text.blocks.15.attn.qkv.weight": "model.safetensors",
+    "text.blocks.16.mlp.fc1_q": "model.safetensors",
+    "text.blocks.16.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.16.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.16.attn.tau_alpha": "model.safetensors",
+    "text.blocks.14.attn.tau_wq": "model.safetensors",
+    "text.blocks.16.ln.bias": "model.safetensors",
+    "text.blocks.14.attn.qkv.bias": "model.safetensors",
+    "text.blocks.18.ln.bias": "model.safetensors",
+    "text.blocks.18.attn.tau_wv": "model.safetensors",
+    "text.blocks.14.attn.proj.weight": "model.safetensors",
+    "text.blocks.19.attn.proj.weight": "model.safetensors",
+    "text.blocks.15.ln.bias": "model.safetensors",
+    "text.blocks.17.mlp.router.bias": "model.safetensors",
+    "text.blocks.15.attn.proj.bias": "model.safetensors",
+    "text.blocks.19.attn.proj.bias": "model.safetensors",
+    "text.blocks.16.attn.tau_wq": "model.safetensors",
+    "text.blocks.14.attn.proj.bias": "model.safetensors",
+    "text.blocks.14.mlp.fc2_q": "model.safetensors",
+    "text.blocks.14.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.14.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.16.attn.qkv.bias": "model.safetensors",
+    "text.blocks.18.mlp.router.weight": "model.safetensors",
+    "text.blocks.14.attn.tau_alpha": "model.safetensors",
+    "text.blocks.19.attn.tau_wv": "model.safetensors",
+    "text.blocks.15.attn.proj.weight": "model.safetensors",
+    "text.blocks.14.mlp.router.bias": "model.safetensors",
+    "text.post_ln.bias": "model.safetensors",
+    "text.blocks.23.mlp.router.bias": "model.safetensors",
+    "text.blocks.23.mlp.fc1_q": "model.safetensors",
+    "text.blocks.23.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.23.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.23.ln.weight": "model.safetensors",
+    "text.blocks.23.attn.tau_wq": "model.safetensors",
+    "text.blocks.23.attn.tau_alpha": "model.safetensors",
+    "text.blocks.23.attn.qkv.weight": "model.safetensors",
+    "text.blocks.23.attn.proj.weight": "model.safetensors",
+    "text.blocks.23.attn.proj.bias": "model.safetensors",
+    "text.blocks.22.mlp.router.bias": "model.safetensors",
+    "text.blocks.22.mlp.fc2_q": "model.safetensors",
+    "text.blocks.22.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.22.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.22.mlp.fc1_q": "model.safetensors",
+    "text.blocks.22.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.22.mlp.fc1_biases": "model.safetensors",
+    "region.size_features": "model.safetensors",
+    "text.blocks.22.ln.weight": "model.safetensors",
+    "text.blocks.23.ln.bias": "model.safetensors",
+    "text.blocks.22.attn.qkv.weight": "model.safetensors",
+    "text.lm_head.bias": "model.safetensors",
+    "text.blocks.22.attn.qkv.bias": "model.safetensors",
+    "text.blocks.22.attn.proj.weight": "model.safetensors",
+    "text.blocks.22.attn.proj.bias": "model.safetensors",
+    "region.coord_decoder.bias": "model.safetensors",
+    "text.blocks.21.mlp.router.bias": "model.safetensors",
+    "text.blocks.21.mlp.fc2_q": "model.safetensors",
+    "text.blocks.21.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.21.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.21.ln.bias": "model.safetensors",
+    "text.blocks.21.attn.qkv.weight": "model.safetensors",
+    "text.blocks.21.attn.qkv.bias": "model.safetensors",
+    "text.blocks.22.attn.tau_wv": "model.safetensors",
+    "text.blocks.21.attn.proj.bias": "model.safetensors",
+    "text.blocks.20.mlp.router.bias": "model.safetensors",
+    "text.post_ln.weight": "model.safetensors",
+    "region.size_encoder.bias": "model.safetensors",
+    "text.blocks.20.mlp.fc2_q": "model.safetensors",
+    "text.blocks.20.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.20.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.20.mlp.fc1_q": "model.safetensors",
+    "text.blocks.20.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.20.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.21.mlp.router.weight": "model.safetensors",
+    "text.blocks.20.ln.bias": "model.safetensors",
+    "text.blocks.21.attn.tau_alpha": "model.safetensors",
+    "text.blocks.20.attn.tau_wq": "model.safetensors",
+    "text.blocks.21.mlp.fc1_q": "model.safetensors",
+    "text.blocks.21.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.21.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.20.attn.tau_alpha": "model.safetensors",
+    "text.blocks.23.mlp.fc2_q": "model.safetensors",
+    "text.blocks.23.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.23.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.23.attn.qkv.bias": "model.safetensors",
+    "text.blocks.21.attn.tau_wv": "model.safetensors",
+    "region.size_decoder.bias": "model.safetensors",
+    "text.blocks.20.attn.tau_wv": "model.safetensors",
+    "text.blocks.20.ln.weight": "model.safetensors",
+    "text.blocks.23.mlp.router.weight": "model.safetensors",
+    "text.blocks.21.ln.weight": "model.safetensors",
+    "text.blocks.20.mlp.router.weight": "model.safetensors",
+    "text.blocks.20.attn.qkv.weight": "model.safetensors",
+    "text.blocks.23.attn.tau_wv": "model.safetensors",
+    "text.blocks.22.ln.bias": "model.safetensors",
+    "region.size_encoder.weight": "model.safetensors",
+    "text.blocks.20.attn.proj.bias": "model.safetensors",
+    "region.coord_encoder.bias": "model.safetensors",
+    "region.coord_encoder.weight": "model.safetensors",
+    "text.blocks.19.mlp.fc2_q": "model.safetensors",
+    "text.blocks.19.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.19.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.19.mlp.fc1_q": "model.safetensors",
+    "text.blocks.19.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.19.mlp.fc1_biases": "model.safetensors",
+    "text.lm_head.weight": "model.safetensors",
+    "text.blocks.22.attn.tau_wq": "model.safetensors",
+    "region.coord_features": "model.safetensors",
+    "region.size_decoder.weight": "model.safetensors",
+    "text.blocks.20.attn.proj.weight": "model.safetensors",
+    "text.blocks.22.mlp.router.weight": "model.safetensors",
+    "text.blocks.21.attn.tau_wq": "model.safetensors",
+    "text.blocks.21.attn.proj.weight": "model.safetensors",
+    "text.blocks.20.attn.qkv.bias": "model.safetensors",
+    "text.blocks.22.attn.tau_alpha": "model.safetensors",
+    "region.coord_decoder.weight": "model.safetensors",
+    "vision.proj_fc2.bias": "model.safetensors",
+    "vision.proj_fc1.bias": "model.safetensors",
+    "vision.post_ln.weight": "model.safetensors",
+    "vision.pos_emb": "model.safetensors",
+    "vision.patch_emb.weight": "model.safetensors",
+    "vision.patch_emb.bias": "model.safetensors",
+    "vision.blocks.9.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.9.ln2.weight": "model.safetensors",
+    "vision.blocks.9.ln2.bias": "model.safetensors",
+    "vision.blocks.9.ln1.bias": "model.safetensors",
+    "vision.blocks.9.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.9.attn.proj.bias": "model.safetensors",
+    "vision.blocks.8.ln2.weight": "model.safetensors",
+    "vision.blocks.8.ln2.bias": "model.safetensors",
+    "vision.blocks.8.attn.proj.weight": "model.safetensors",
+    "vision.blocks.7.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.7.ln2.weight": "model.safetensors",
+    "vision.blocks.7.ln1.weight": "model.safetensors",
+    "vision.blocks.7.ln1.bias": "model.safetensors",
+    "vision.blocks.7.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.7.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.7.attn.proj.bias": "model.safetensors",
+    "vision.blocks.9.ln1.weight": "model.safetensors",
+    "vision.blocks.6.ln2.weight": "model.safetensors",
+    "vision.blocks.6.ln1.weight": "model.safetensors",
+    "vision.blocks.2.ln1.bias": "model.safetensors",
+    "vision.blocks.6.ln1.bias": "model.safetensors",
+    "vision.blocks.16.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.6.attn.proj.weight": "model.safetensors",
+    "vision.blocks.10.attn.proj.weight": "model.safetensors",
+    "vision.blocks.5.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.18.ln1.bias": "model.safetensors",
+    "vision.blocks.5.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.5.ln2.bias": "model.safetensors",
+    "vision.blocks.5.ln1.bias": "model.safetensors",
+    "text.blocks.4.ln.bias": "model.safetensors",
+    "vision.blocks.5.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.11.ln1.weight": "model.safetensors",
+    "vision.blocks.4.ln2.bias": "model.safetensors",
+    "vision.blocks.14.attn.proj.weight": "model.safetensors",
+    "vision.blocks.12.attn.proj.bias": "model.safetensors",
+    "vision.blocks.4.ln1.weight": "model.safetensors",
+    "vision.blocks.5.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.4.ln1.bias": "model.safetensors",
+    "vision.blocks.4.attn.proj.bias": "model.safetensors",
+    "text.blocks.6.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.22.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.3.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.13.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.3.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.8.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.3.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.13.attn.proj.weight": "model.safetensors",
+    "text.blocks.3.attn.tau_wv": "model.safetensors",
+    "vision.blocks.3.ln2.bias": "model.safetensors",
+    "text.blocks.7.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.3.ln1.weight": "model.safetensors",
+    "vision.blocks.3.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.16.ln1.weight": "model.safetensors",
+    "vision.blocks.3.attn.proj.weight": "model.safetensors",
+    "vision.blocks.3.attn.proj.bias": "model.safetensors",
+    "vision.blocks.26.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.26.mlp.fc2.bias": "model.safetensors",
+    "text.blocks.3.ln.bias": "model.safetensors",
+    "vision.blocks.24.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.26.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.3.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.12.attn.proj.weight": "model.safetensors",
+    "vision.blocks.26.mlp.fc1.bias": "model.safetensors",
+    "text.blocks.3.attn.tau_wq": "model.safetensors",
+    "vision.blocks.26.ln2.weight": "model.safetensors",
+    "vision.blocks.8.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.16.ln1.bias": "model.safetensors",
+    "vision.blocks.26.ln2.bias": "model.safetensors",
+    "vision.blocks.26.ln1.weight": "model.safetensors",
+    "vision.blocks.26.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.26.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.7.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.10.ln2.bias": "model.safetensors",
+    "vision.blocks.26.attn.proj.weight": "model.safetensors",
+    "vision.blocks.8.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.1.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.5.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.11.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.25.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.25.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.25.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.20.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.25.ln2.bias": "model.safetensors",
+    "text.blocks.4.attn.proj.bias": "model.safetensors",
+    "text.blocks.8.attn.proj.bias": "model.safetensors",
+    "vision.blocks.25.attn.proj.bias": "model.safetensors",
+    "vision.blocks.5.ln1.weight": "model.safetensors",
+    "vision.blocks.0.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.22.attn.proj.bias": "model.safetensors",
+    "vision.blocks.24.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.24.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.24.ln2.bias": "model.safetensors",
+    "vision.blocks.24.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.5.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.24.attn.qkv.bias": "model.safetensors",
+    "text.blocks.4.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.10.attn.proj.bias": "model.safetensors",
+    "vision.blocks.4.attn.proj.weight": "model.safetensors",
+    "vision.blocks.24.attn.proj.bias": "model.safetensors",
+    "vision.blocks.23.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.1.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.21.ln2.weight": "model.safetensors",
+    "vision.blocks.6.attn.proj.bias": "model.safetensors",
+    "vision.blocks.23.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.12.ln2.bias": "model.safetensors",
+    "vision.blocks.23.ln1.weight": "model.safetensors",
+    "vision.blocks.23.attn.proj.bias": "model.safetensors",
+    "vision.blocks.8.ln1.weight": "model.safetensors",
+    "vision.blocks.25.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.25.ln1.weight": "model.safetensors",
+    "vision.blocks.22.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.9.attn.proj.weight": "model.safetensors",
+    "vision.blocks.22.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.22.ln2.weight": "model.safetensors",
+    "vision.blocks.22.ln2.bias": "model.safetensors",
+    "vision.blocks.22.attn.proj.weight": "model.safetensors",
+    "text.blocks.7.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.21.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.21.ln1.bias": "model.safetensors",
+    "vision.blocks.11.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.21.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.9.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.21.attn.qkv.weight": "model.safetensors",
+    "text.blocks.6.attn.tau_wv": "model.safetensors",
+    "vision.blocks.20.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.21.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.20.ln2.weight": "model.safetensors",
+    "vision.blocks.20.attn.proj.bias": "model.safetensors",
+    "vision.blocks.19.ln2.weight": "model.safetensors",
+    "vision.blocks.23.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.2.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.20.attn.proj.weight": "model.safetensors",
+    "vision.blocks.22.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.2.mlp.fc2.bias": "model.safetensors",
+    "text.blocks.6.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.2.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.2.attn.proj.bias": "model.safetensors",
+    "text.blocks.1.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.4.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.19.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.1.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.19.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.19.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.18.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.5.attn.proj.weight": "model.safetensors",
+    "vision.blocks.18.ln1.weight": "model.safetensors",
+    "vision.blocks.19.ln1.weight": "model.safetensors",
+    "vision.blocks.19.ln1.bias": "model.safetensors",
+    "vision.blocks.18.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.13.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.21.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.21.ln2.bias": "model.safetensors",
+    "text.blocks.5.attn.proj.bias": "model.safetensors",
+    "vision.blocks.18.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.3.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.18.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.18.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.18.ln2.weight": "model.safetensors",
+    "vision.blocks.20.ln1.weight": "model.safetensors",
+    "vision.blocks.19.attn.proj.bias": "model.safetensors",
+    "vision.blocks.2.ln1.weight": "model.safetensors",
+    "vision.blocks.18.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.23.ln1.bias": "model.safetensors",
+    "vision.blocks.18.attn.proj.bias": "model.safetensors",
+    "vision.blocks.17.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.2.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.17.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.6.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.17.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.17.ln2.bias": "model.safetensors",
+    "vision.blocks.17.ln1.bias": "model.safetensors",
+    "text.blocks.5.ln.bias": "model.safetensors",
+    "vision.blocks.23.ln2.bias": "model.safetensors",
+    "vision.blocks.6.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.17.attn.proj.bias": "model.safetensors",
+    "vision.blocks.16.mlp.fc2.weight": "model.safetensors",
+    "vision.post_ln.bias": "model.safetensors",
+    "vision.blocks.16.ln2.weight": "model.safetensors",
+    "vision.blocks.16.ln2.bias": "model.safetensors",
+    "vision.blocks.21.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.16.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.25.ln2.weight": "model.safetensors",
+    "text.blocks.3.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.16.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.16.attn.proj.bias": "model.safetensors",
+    "vision.blocks.2.attn.proj.weight": "model.safetensors",
+    "vision.blocks.16.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.15.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.8.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.15.ln2.weight": "model.safetensors",
+    "vision.blocks.6.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.15.ln2.bias": "model.safetensors",
+    "vision.blocks.15.ln1.weight": "model.safetensors",
+    "vision.proj_fc1.weight": "model.safetensors",
+    "vision.blocks.18.ln2.bias": "model.safetensors",
+    "vision.blocks.23.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.24.ln2.weight": "model.safetensors",
+    "vision.blocks.15.ln1.bias": "model.safetensors",
+    "vision.blocks.15.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.15.attn.proj.bias": "model.safetensors",
+    "vision.blocks.14.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.14.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.2.attn.proj.bias": "model.safetensors",
+    "vision.blocks.14.ln2.bias": "model.safetensors",
+    "text.blocks.4.attn.proj.weight": "model.safetensors",
+    "vision.blocks.14.ln2.weight": "model.safetensors",
+    "vision.blocks.10.attn.qkv.bias": "model.safetensors",
+    "text.blocks.3.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.14.attn.qkv.weight": "model.safetensors",
+    "text.blocks.5.mlp.fc2_q": "model.safetensors",
+    "text.blocks.5.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.5.mlp.fc2_biases": "model.safetensors",
+    "vision.blocks.6.attn.qkv.weight": "model.safetensors",
+    "text.blocks.3.ln.weight": "model.safetensors",
+    "vision.blocks.26.attn.proj.bias": "model.safetensors",
+    "text.blocks.0.ln.bias": "model.safetensors",
+    "vision.blocks.14.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.3.ln1.bias": "model.safetensors",
+    "vision.blocks.23.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.21.attn.proj.weight": "model.safetensors",
+    "vision.blocks.12.ln1.weight": "model.safetensors",
+    "vision.blocks.14.attn.proj.bias": "model.safetensors",
+    "vision.blocks.5.attn.proj.bias": "model.safetensors",
+    "vision.blocks.13.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.2.attn.qkv.bias": "model.safetensors",
+    "text.blocks.3.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.0.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.13.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.23.attn.proj.weight": "model.safetensors",
+    "vision.blocks.7.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.17.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.13.ln1.weight": "model.safetensors",
+    "vision.blocks.1.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.13.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.12.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.13.ln2.weight": "model.safetensors",
+    "vision.blocks.12.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.20.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.12.ln1.bias": "model.safetensors",
+    "vision.blocks.12.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.5.ln2.weight": "model.safetensors",
+    "text.blocks.8.mlp.router.bias": "model.safetensors",
+    "vision.blocks.11.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.11.ln2.weight": "model.safetensors",
+    "vision.blocks.11.ln2.bias": "model.safetensors",
+    "text.blocks.5.attn.tau_wv": "model.safetensors",
+    "vision.blocks.11.ln1.bias": "model.safetensors",
+    "text.blocks.8.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.11.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.11.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.16.attn.proj.weight": "model.safetensors",
+    "text.blocks.5.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.0.attn.proj.weight": "model.safetensors",
+    "vision.blocks.7.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.10.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.2.ln2.weight": "model.safetensors",
+    "vision.blocks.10.mlp.fc2.bias": "model.safetensors",
+    "text.blocks.8.mlp.router.weight": "model.safetensors",
+    "vision.blocks.9.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.10.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.6.mlp.router.weight": "model.safetensors",
+    "text.blocks.6.attn.proj.weight": "model.safetensors",
+    "vision.blocks.10.ln2.weight": "model.safetensors",
+    "vision.blocks.10.ln1.bias": "model.safetensors",
+    "vision.blocks.16.mlp.fc2.bias": "model.safetensors",
+    "text.blocks.0.attn.tau_alpha": "model.safetensors",
+    "vision.blocks.12.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.22.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.1.ln2.weight": "model.safetensors",
+    "vision.blocks.15.mlp.fc2.weight": "model.safetensors",
+    "text.blocks.4.mlp.router.bias": "model.safetensors",
+    "vision.blocks.1.ln1.bias": "model.safetensors",
+    "vision.blocks.1.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.23.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.0.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.9.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.18.attn.proj.weight": "model.safetensors",
+    "vision.blocks.4.ln2.weight": "model.safetensors",
+    "text.blocks.6.mlp.router.bias": "model.safetensors",
+    "vision.blocks.17.attn.proj.weight": "model.safetensors",
+    "vision.blocks.0.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.11.attn.proj.weight": "model.safetensors",
+    "text.blocks.4.mlp.router.weight": "model.safetensors",
+    "vision.blocks.1.ln2.bias": "model.safetensors",
+    "vision.blocks.1.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.0.ln1.weight": "model.safetensors",
+    "text.blocks.0.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.0.ln2.bias": "model.safetensors",
+    "vision.blocks.4.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.0.ln1.bias": "model.safetensors",
+    "text.blocks.6.ln.bias": "model.safetensors",
+    "text.blocks.1.ln.weight": "model.safetensors",
+    "vision.blocks.11.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.0.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.0.attn.qkv.bias": "model.safetensors",
+    "text.blocks.8.ln.weight": "model.safetensors",
+    "vision.blocks.25.attn.proj.weight": "model.safetensors",
+    "text.wte.weight": "model.safetensors",
+    "text.blocks.8.attn.tau_wv": "model.safetensors",
+    "vision.blocks.19.ln2.bias": "model.safetensors",
+    "vision.blocks.24.attn.proj.weight": "model.safetensors",
+    "text.blocks.8.attn.tau_wq": "model.safetensors",
+    "vision.blocks.22.ln1.weight": "model.safetensors",
+    "text.blocks.0.attn.tau_wv": "model.safetensors",
+    "vision.blocks.8.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.6.attn.qkv.bias": "model.safetensors",
+    "text.blocks.8.attn.qkv.bias": "model.safetensors",
+    "text.blocks.1.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.12.ln2.weight": "model.safetensors",
+    "text.blocks.6.attn.tau_alpha": "model.safetensors",
+    "text.blocks.8.attn.proj.weight": "model.safetensors",
+    "vision.blocks.17.attn.qkv.weight": "model.safetensors",
+    "text.blocks.7.attn.tau_wq": "model.safetensors",
+    "text.blocks.7.mlp.router.bias": "model.safetensors",
+    "vision.proj_fc2.weight": "model.safetensors",
+    "text.blocks.7.ln.bias": "model.safetensors",
+    "text.blocks.0.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.10.ln1.weight": "model.safetensors",
+    "text.blocks.2.mlp.fc2.bias": "model.safetensors",
+    "text.blocks.1.attn.proj.weight": "model.safetensors",
+    "vision.blocks.14.ln1.weight": "model.safetensors",
+    "vision.blocks.19.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.8.ln1.bias": "model.safetensors",
+    "text.blocks.6.ln.weight": "model.safetensors",
+    "vision.blocks.24.mlp.fc2.weight": "model.safetensors",
+    "text.blocks.6.mlp.fc1_q": "model.safetensors",
+    "text.blocks.6.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.6.mlp.fc1_biases": "model.safetensors",
+    "vision.blocks.8.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.15.attn.qkv.bias": "model.safetensors",
+    "text.blocks.7.mlp.router.weight": "model.safetensors",
+    "text.blocks.7.attn.proj.weight": "model.safetensors",
+    "vision.blocks.1.attn.proj.weight": "model.safetensors",
+    "vision.blocks.7.attn.proj.weight": "model.safetensors",
+    "vision.blocks.19.mlp.fc2.weight": "model.safetensors",
+    "text.blocks.3.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.7.ln.weight": "model.safetensors",
+    "text.blocks.1.attn.qkv.weight": "model.safetensors",
+    "text.blocks.5.mlp.router.weight": "model.safetensors",
+    "vision.blocks.21.attn.proj.bias": "model.safetensors",
+    "text.blocks.5.mlp.router.bias": "model.safetensors",
+    "text.blocks.4.ln.weight": "model.safetensors",
+    "vision.blocks.2.attn.qkv.weight": "model.safetensors",
+    "text.blocks.5.attn.tau_wq": "model.safetensors",
+    "vision.blocks.13.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.3.ln2.weight": "model.safetensors",
+    "vision.blocks.10.attn.qkv.weight": "model.safetensors",
+    "text.blocks.3.attn.qkv.bias": "model.safetensors",
+    "text.blocks.3.attn.qkv.weight": "model.safetensors",
+    "vision.blocks.1.ln1.weight": "model.safetensors",
+    "text.blocks.8.attn.tau_alpha": "model.safetensors",
+    "text.blocks.6.attn.tau_wq": "model.safetensors",
+    "text.blocks.7.mlp.fc2_q": "model.safetensors",
+    "text.blocks.7.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.7.mlp.fc2_biases": "model.safetensors",
+    "vision.blocks.25.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.17.ln2.weight": "model.safetensors",
+    "text.blocks.2.mlp.fc1.bias": "model.safetensors",
+    "text.blocks.6.mlp.fc2_q": "model.safetensors",
+    "text.blocks.6.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.6.mlp.fc2_biases": "model.safetensors",
+    "vision.blocks.17.ln1.weight": "model.safetensors",
+    "vision.blocks.14.mlp.fc1.bias": "model.safetensors",
+    "text.blocks.4.attn.tau_wv": "model.safetensors",
+    "vision.blocks.7.ln2.bias": "model.safetensors",
+    "text.blocks.4.attn.tau_alpha": "model.safetensors",
+    "vision.blocks.22.ln1.bias": "model.safetensors",
+    "vision.blocks.20.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.13.ln1.bias": "model.safetensors",
+    "vision.blocks.23.ln2.weight": "model.safetensors",
+    "text.blocks.2.ln.bias": "model.safetensors",
+    "text.blocks.5.attn.proj.weight": "model.safetensors",
+    "vision.blocks.22.attn.qkv.weight": "model.safetensors",
+    "text.blocks.5.mlp.fc1_q": "model.safetensors",
+    "text.blocks.5.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.5.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.5.attn.tau_alpha": "model.safetensors",
+    "text.blocks.4.mlp.fc1_q": "model.safetensors",
+    "text.blocks.4.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.4.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.0.attn.proj.bias": "model.safetensors",
+    "text.blocks.1.attn.tau_alpha": "model.safetensors",
+    "text.blocks.3.attn.proj.bias": "model.safetensors",
+    "vision.blocks.13.attn.proj.bias": "model.safetensors",
+    "vision.blocks.25.ln1.bias": "model.safetensors",
+    "vision.blocks.24.ln1.bias": "model.safetensors",
+    "text.blocks.2.ln.weight": "model.safetensors",
+    "vision.blocks.20.ln1.bias": "model.safetensors",
+    "vision.blocks.8.attn.proj.bias": "model.safetensors",
+    "vision.blocks.10.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.19.attn.proj.weight": "model.safetensors",
+    "text.blocks.2.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.20.attn.qkv.weight": "model.safetensors",
+    "text.blocks.7.attn.proj.bias": "model.safetensors",
+    "vision.blocks.12.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.2.attn.tau_wv": "model.safetensors",
+    "text.blocks.3.attn.tau_alpha": "model.safetensors",
+    "text.blocks.5.attn.qkv.weight": "model.safetensors",
+    "text.blocks.2.attn.tau_wq": "model.safetensors",
+    "vision.blocks.20.ln2.bias": "model.safetensors",
+    "text.blocks.0.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.20.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.9.mlp.fc2.bias": "model.safetensors",
+    "vision.blocks.19.attn.qkv.weight": "model.safetensors",
+    "text.blocks.2.attn.qkv.weight": "model.safetensors",
+    "text.blocks.7.attn.tau_alpha": "model.safetensors",
+    "text.blocks.7.attn.tau_wv": "model.safetensors",
+    "vision.blocks.14.ln1.bias": "model.safetensors",
+    "vision.blocks.0.ln2.weight": "model.safetensors",
+    "text.blocks.2.attn.proj.weight": "model.safetensors",
+    "vision.blocks.13.ln2.bias": "model.safetensors",
+    "text.blocks.0.attn.tau_wq": "model.safetensors",
+    "vision.blocks.4.attn.qkv.weight": "model.safetensors",
+    "text.blocks.4.attn.tau_wq": "model.safetensors",
+    "vision.blocks.25.attn.qkv.bias": "model.safetensors",
+    "text.blocks.0.ln.weight": "model.safetensors",
+    "text.blocks.4.attn.qkv.weight": "model.safetensors",
+    "text.blocks.1.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.2.ln2.bias": "model.safetensors",
+    "vision.blocks.15.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.0.attn.proj.bias": "model.safetensors",
+    "vision.blocks.4.attn.qkv.bias": "model.safetensors",
+    "vision.blocks.6.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.2.attn.tau_alpha": "model.safetensors",
+    "vision.blocks.1.mlp.fc1.weight": "model.safetensors",
+    "text.blocks.6.attn.proj.bias": "model.safetensors",
+    "vision.blocks.12.mlp.fc1.bias": "model.safetensors",
+    "text.blocks.4.mlp.fc2_q": "model.safetensors",
+    "text.blocks.4.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.4.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.1.attn.qkv.bias": "model.safetensors",
+    "text.blocks.2.mlp.fc2.weight": "model.safetensors",
+    "vision.blocks.26.ln1.bias": "model.safetensors",
+    "vision.blocks.21.ln1.weight": "model.safetensors",
+    "text.blocks.8.ln.bias": "model.safetensors",
+    "text.blocks.0.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.15.mlp.fc1.bias": "model.safetensors",
+    "vision.blocks.1.attn.proj.bias": "model.safetensors",
+    "text.blocks.5.ln.weight": "model.safetensors",
+    "vision.blocks.15.attn.proj.weight": "model.safetensors",
+    "text.blocks.1.attn.proj.bias": "model.safetensors",
+    "text.blocks.3.attn.proj.weight": "model.safetensors",
+    "vision.blocks.11.attn.proj.bias": "model.safetensors",
+    "vision.blocks.6.ln2.bias": "model.safetensors",
+    "text.blocks.1.attn.tau_wq": "model.safetensors",
+    "text.blocks.0.attn.qkv.weight": "model.safetensors",
+    "text.blocks.1.attn.tau_wv": "model.safetensors",
+    "text.blocks.1.ln.bias": "model.safetensors",
+    "vision.blocks.4.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.17.mlp.fc1.weight": "model.safetensors",
+    "vision.blocks.14.mlp.fc2.weight": "model.safetensors",
+    "text.blocks.7.mlp.fc1_q": "model.safetensors",
+    "text.blocks.7.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.7.mlp.fc1_biases": "model.safetensors",
+    "vision.blocks.4.mlp.fc1.bias": "model.safetensors",
+    "text.blocks.0.attn.proj.weight": "model.safetensors",
+    "vision.blocks.24.ln1.weight": "model.safetensors",
+    "text.blocks.2.attn.qkv.bias": "model.safetensors",
+    "text.blocks.0.attn.qkv.bias": "model.safetensors",
+    "text.blocks.9.mlp.router.weight": "model.safetensors",
+    "text.blocks.9.ln.bias": "model.safetensors",
+    "text.blocks.9.attn.tau_wv": "model.safetensors",
+    "text.blocks.9.attn.tau_wq": "model.safetensors",
+    "text.blocks.9.attn.qkv.weight": "model.safetensors",
+    "text.blocks.9.attn.qkv.bias": "model.safetensors",
+    "text.blocks.9.attn.proj.bias": "model.safetensors",
+    "text.blocks.8.mlp.fc2_q": "model.safetensors",
+    "text.blocks.8.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.8.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.8.mlp.fc1_q": "model.safetensors",
+    "text.blocks.8.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.8.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.13.mlp.router.bias": "model.safetensors",
+    "text.blocks.13.ln.weight": "model.safetensors",
+    "text.blocks.10.mlp.router.bias": "model.safetensors",
+    "text.blocks.12.mlp.router.weight": "model.safetensors",
+    "text.blocks.11.ln.bias": "model.safetensors",
+    "text.blocks.12.mlp.fc2_q": "model.safetensors",
+    "text.blocks.12.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.12.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.13.attn.proj.bias": "model.safetensors",
+    "text.blocks.9.mlp.fc2_q": "model.safetensors",
+    "text.blocks.9.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.9.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.12.mlp.fc1_q": "model.safetensors",
+    "text.blocks.12.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.12.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.9.ln.weight": "model.safetensors",
+    "text.blocks.12.ln.weight": "model.safetensors",
+    "text.blocks.13.attn.tau_wv": "model.safetensors",
+    "text.blocks.12.attn.tau_wv": "model.safetensors",
+    "text.blocks.12.attn.tau_alpha": "model.safetensors",
+    "text.blocks.10.attn.proj.bias": "model.safetensors",
+    "text.blocks.13.ln.bias": "model.safetensors",
+    "text.blocks.12.attn.qkv.weight": "model.safetensors",
+    "text.blocks.12.attn.qkv.bias": "model.safetensors",
+    "text.blocks.12.attn.proj.weight": "model.safetensors",
+    "text.blocks.10.mlp.router.weight": "model.safetensors",
+    "text.blocks.13.attn.qkv.weight": "model.safetensors",
+    "text.blocks.12.attn.proj.bias": "model.safetensors",
+    "text.blocks.13.mlp.fc1_q": "model.safetensors",
+    "text.blocks.13.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.13.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.11.mlp.router.bias": "model.safetensors",
+    "text.blocks.10.mlp.fc2_q": "model.safetensors",
+    "text.blocks.10.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.10.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.13.attn.proj.weight": "model.safetensors",
+    "text.blocks.11.mlp.fc2_q": "model.safetensors",
+    "text.blocks.11.mlp.fc2_scales": "model.safetensors",
+    "text.blocks.11.mlp.fc2_biases": "model.safetensors",
+    "text.blocks.9.attn.tau_alpha": "model.safetensors",
+    "text.blocks.11.attn.proj.bias": "model.safetensors",
+    "text.blocks.10.ln.bias": "model.safetensors",
+    "text.blocks.11.mlp.fc1_q": "model.safetensors",
+    "text.blocks.11.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.11.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.11.attn.qkv.weight": "model.safetensors",
+    "text.blocks.11.attn.proj.weight": "model.safetensors",
+    "text.blocks.9.mlp.router.bias": "model.safetensors",
+    "text.blocks.11.attn.tau_wq": "model.safetensors",
+    "text.blocks.10.attn.tau_wv": "model.safetensors",
+    "text.blocks.11.attn.tau_alpha": "model.safetensors",
+    "text.blocks.12.ln.bias": "model.safetensors",
+    "text.blocks.9.mlp.fc1_q": "model.safetensors",
+    "text.blocks.9.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.9.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.9.attn.proj.weight": "model.safetensors",
+    "text.blocks.11.mlp.router.weight": "model.safetensors",
+    "text.blocks.11.attn.qkv.bias": "model.safetensors",
+    "text.blocks.13.attn.tau_alpha": "model.safetensors",
+    "text.blocks.10.mlp.fc1_q": "model.safetensors",
+    "text.blocks.10.mlp.fc1_scales": "model.safetensors",
+    "text.blocks.10.mlp.fc1_biases": "model.safetensors",
+    "text.blocks.10.attn.qkv.weight": "model.safetensors",
+    "text.blocks.11.ln.weight": "model.safetensors",
+    "text.blocks.10.ln.weight": "model.safetensors",
+    "text.blocks.13.attn.qkv.bias": "model.safetensors",
+    "text.blocks.12.mlp.router.bias": "model.safetensors",
+    "text.blocks.12.attn.tau_wq": "model.safetensors",
+    "text.blocks.10.attn.tau_wq": "model.safetensors",
+    "text.blocks.11.attn.tau_wv": "model.safetensors",
+    "text.blocks.10.attn.qkv.bias": "model.safetensors",
+    "text.blocks.13.mlp.router.weight": "model.safetensors",
+    "text.blocks.13.attn.tau_wq": "model.safetensors",
+    "text.blocks.10.attn.proj.weight": "model.safetensors",
+    "text.blocks.10.attn.tau_alpha": "model.safetensors"
+  }
+}