Upload folder using huggingface_hub

convert_molmo_point_to_hf.py

@@ -204,9 +204,9 @@ def convert_molmo2(
     new_state_dict = {}
     for key, val in state_dict.items():
         if key == "transformer.ff_out.new_weight":
-            new_key = "new_output_embeddings"
+            new_key = "lm_head.new_output_embeddings"
         elif key == "transformer.ff_out.weight":
-            new_key = "output_embeddings"
+            new_key = "lm_head.output_embeddings"
         else:
             new_key = f"{base_model_prefix}.{key}"
         new_state_dict[new_key] = val

model-00001-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f3e72190ef32e8a730642be569f673e3db747342d1233337b8839f805c02ce8
-size 4982833608
+oid sha256:19ab63a529d72b101000396a03df34f2dc32e8d744b0c5cec50f0542eea6b8db
+size 4974567112

model-00002-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:396c71733dec4db53150621ba67aea1c8444e3cbdc06410427ee52c9b34090ff
-size 4798510440
+oid sha256:19fbea500ac2cb32175131e3009b4aebfc3c373e48b07ea51783b1049df34761
+size 4630720272

model-00003-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a8d305edb19cd8bae64920bc893e636612e4722e986558338b097246b84efd6
-size 4630720272
+oid sha256:c66ec2e00cd832d67a5f24ace0e4330e13627b82444784986062df6a2a973ac4
+size 4630720296

model-00004-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e09b72e931382e8a0d7566ce791059f0d1e307358ff445aeab940083ae5b8ba2
+oid sha256:20b4298b55a145d9ccdef463ffe3c344cbfd90e7fa67d0778d50f66688dacd27
 size 4630720320

model-00005-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d583e6ccfd62c1a710bd6405ce81ae965612fe814019004fd9357a4f2810f91
+oid sha256:eeac368f9670bad559e45aceea971ff78db86fafee604f17e28c1e58932a27de
 size 4630720320

model-00006-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:957fd289c1ce380f606ac7c66fcf88fd071e59359e1824b34c56e040f118a302
+oid sha256:3738d142536fdc8e9217e7a853240006aff3fe5c157d982cff3e52addb05f011
 size 4630720320

model-00007-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f14ba921030350d6277d794066529298e9db125b763f109c17037611c5f379cb
-size 4997804128
+oid sha256:30485f5c86cefbfccb5eef21be09adf1149e4ec6f9cc75072c162900e6972226
+size 4091924852

model-00008-of-00008.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3727a21ae90c0f0fc3e59f9a42354db0edafa6bc8577baed9b7c362446b15dd4
-size 1409480580
+oid sha256:b87b59f8beb9a114e4867e522e3ec5116d05442f199901b48b1beb4fda6962d1
+size 2491416816

model.safetensors.index.json

@@ -4,44 +4,46 @@
     "total_size": 34711420260
   },
   "weight_map": {
-    "model.add_no_point_class_embed.vector": "model-00008-of-00008.safetensors",
-    "model.build_vit_embedding.bias": "model-00008-of-00008.safetensors",
-    "model.build_vit_embedding.weight": "model-00008-of-00008.safetensors",
-    "model.connector.image_pooling_2d.wk.bias": "model-00008-of-00008.safetensors",
-    "model.connector.image_pooling_2d.wk.weight": "model-00008-of-00008.safetensors",
-    "model.connector.image_pooling_2d.wq.bias": "model-00008-of-00008.safetensors",
-    "model.connector.image_pooling_2d.wq.weight": "model-00008-of-00008.safetensors",
-    "model.connector.image_pooling_2d.wv.bias": "model-00008-of-00008.safetensors",
-    "model.connector.image_pooling_2d.wv.weight": "model-00008-of-00008.safetensors",
-    "model.connector.image_projector.w1.weight": "model-00008-of-00008.safetensors",
-    "model.connector.image_projector.w2.weight": "model-00008-of-00008.safetensors",
-    "model.connector.image_projector.w3.weight": "model-00008-of-00008.safetensors",
-    "model.patch_k.bias": "model-00008-of-00008.safetensors",
-    "model.patch_k.weight": "model-00008-of-00008.safetensors",
-    "model.patch_q.bias": "model-00008-of-00008.safetensors",
-    "model.patch_q.weight": "model-00008-of-00008.safetensors",
-    "model.subpatch_k.bias": "model-00008-of-00008.safetensors",
-    "model.subpatch_k.weight": "model-00008-of-00008.safetensors",
-    "model.subpatch_loc_k.bias": "model-00008-of-00008.safetensors",
-    "model.subpatch_loc_k.weight": "model-00008-of-00008.safetensors",
-    "model.subpatch_q.bias": "model-00008-of-00008.safetensors",
-    "model.subpatch_q.weight": "model-00008-of-00008.safetensors",
-    "model.transformer.blocks.0.attn_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.ff_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.0.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.attn_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.ff_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.1.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
+    "lm_head.new_output_embeddings": "model-00008-of-00008.safetensors",
+    "lm_head.output_embeddings": "model-00008-of-00008.safetensors",
+    "model.add_no_point_class_embed.vector": "model-00007-of-00008.safetensors",
+    "model.build_vit_embedding.bias": "model-00007-of-00008.safetensors",
+    "model.build_vit_embedding.weight": "model-00007-of-00008.safetensors",
+    "model.connector.image_pooling_2d.wk.bias": "model-00007-of-00008.safetensors",
+    "model.connector.image_pooling_2d.wk.weight": "model-00007-of-00008.safetensors",
+    "model.connector.image_pooling_2d.wq.bias": "model-00007-of-00008.safetensors",
+    "model.connector.image_pooling_2d.wq.weight": "model-00007-of-00008.safetensors",
+    "model.connector.image_pooling_2d.wv.bias": "model-00007-of-00008.safetensors",
+    "model.connector.image_pooling_2d.wv.weight": "model-00007-of-00008.safetensors",
+    "model.connector.image_projector.w1.weight": "model-00007-of-00008.safetensors",
+    "model.connector.image_projector.w2.weight": "model-00007-of-00008.safetensors",
+    "model.connector.image_projector.w3.weight": "model-00007-of-00008.safetensors",
+    "model.patch_k.bias": "model-00007-of-00008.safetensors",
+    "model.patch_k.weight": "model-00007-of-00008.safetensors",
+    "model.patch_q.bias": "model-00007-of-00008.safetensors",
+    "model.patch_q.weight": "model-00007-of-00008.safetensors",
+    "model.subpatch_k.bias": "model-00007-of-00008.safetensors",
+    "model.subpatch_k.weight": "model-00007-of-00008.safetensors",
+    "model.subpatch_loc_k.bias": "model-00007-of-00008.safetensors",
+    "model.subpatch_loc_k.weight": "model-00007-of-00008.safetensors",
+    "model.subpatch_q.bias": "model-00007-of-00008.safetensors",
+    "model.subpatch_q.weight": "model-00007-of-00008.safetensors",
+    "model.transformer.blocks.0.attn_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.ff_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.mlp.ff_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.mlp.ff_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.self_attn.att_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.self_attn.attn_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.self_attn.k_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.0.self_attn.q_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.attn_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.ff_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.mlp.ff_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.mlp.ff_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.self_attn.att_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.self_attn.attn_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.self_attn.k_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.1.self_attn.q_norm.weight": "model-00001-of-00008.safetensors",
     "model.transformer.blocks.10.attn_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.10.ff_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.10.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
@@ -59,37 +61,37 @@
     "model.transformer.blocks.11.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.11.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.12.attn_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.12.ff_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.12.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.12.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.12.ff_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.12.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.12.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.12.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.12.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.12.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.12.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.13.attn_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.ff_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.13.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.attn_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.ff_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.14.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.15.attn_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.13.attn_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.ff_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.13.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.attn_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.ff_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.14.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.15.attn_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.15.ff_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.15.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.15.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.15.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.15.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.15.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.15.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.15.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.15.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.15.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.15.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.16.attn_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.16.ff_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.16.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
@@ -107,45 +109,45 @@
     "model.transformer.blocks.17.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.17.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.18.attn_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.18.ff_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.18.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.18.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.18.ff_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.18.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.18.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.18.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.18.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.18.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.18.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
-    "model.transformer.blocks.19.attn_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.ff_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.19.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.2.attn_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.ff_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.2.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.20.attn_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.ff_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.20.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.21.attn_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.19.attn_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.ff_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.19.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.2.attn_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.ff_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.mlp.ff_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.mlp.ff_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.self_attn.att_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.self_attn.attn_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.self_attn.k_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.2.self_attn.q_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.20.attn_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.ff_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.mlp.ff_out.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.mlp.ff_proj.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.20.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.21.attn_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.21.ff_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.21.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.21.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.21.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.21.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.21.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.21.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.21.self_attn.att_proj.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.21.self_attn.attn_out.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.21.self_attn.k_norm.weight": "model-00004-of-00008.safetensors",
+    "model.transformer.blocks.21.self_attn.q_norm.weight": "model-00004-of-00008.safetensors",
     "model.transformer.blocks.22.attn_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.22.ff_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.22.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
@@ -163,37 +165,37 @@
     "model.transformer.blocks.23.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.23.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.24.attn_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.24.ff_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.24.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.24.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.24.ff_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.24.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.24.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.24.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.24.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.24.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.24.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
-    "model.transformer.blocks.25.attn_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.ff_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.25.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.attn_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.ff_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.26.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.27.attn_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.25.attn_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.ff_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.25.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.attn_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.ff_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.mlp.ff_out.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.mlp.ff_proj.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.26.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.27.attn_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.27.ff_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.27.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.27.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.27.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.27.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.27.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.27.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.27.self_attn.att_proj.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.27.self_attn.attn_out.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.27.self_attn.k_norm.weight": "model-00005-of-00008.safetensors",
+    "model.transformer.blocks.27.self_attn.q_norm.weight": "model-00005-of-00008.safetensors",
     "model.transformer.blocks.28.attn_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.28.ff_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.28.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
@@ -210,46 +212,46 @@
     "model.transformer.blocks.29.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.29.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.29.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.3.attn_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.3.attn_norm.weight": "model-00001-of-00008.safetensors",
     "model.transformer.blocks.3.ff_norm.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.3.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.3.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.3.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.3.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.3.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.3.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.3.self_attn.att_proj.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.3.self_attn.attn_out.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.3.self_attn.k_norm.weight": "model-00001-of-00008.safetensors",
+    "model.transformer.blocks.3.self_attn.q_norm.weight": "model-00001-of-00008.safetensors",
     "model.transformer.blocks.30.attn_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.30.ff_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.30.mlp.ff_out.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.30.mlp.ff_proj.weight": "model-00007-of-00008.safetensors",
+    "model.transformer.blocks.30.ff_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.30.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.30.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.30.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.30.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.30.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.30.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
-    "model.transformer.blocks.31.attn_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.ff_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.mlp.ff_out.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.mlp.ff_proj.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.self_attn.att_proj.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.self_attn.attn_out.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.self_attn.k_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.31.self_attn.q_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.attn_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.ff_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.mlp.ff_out.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.mlp.ff_proj.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.self_attn.att_proj.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.self_attn.attn_out.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.self_attn.k_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.32.self_attn.q_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.33.attn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.transformer.blocks.31.attn_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.ff_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.31.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.attn_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.ff_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.mlp.ff_out.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.mlp.ff_proj.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.32.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.33.attn_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.33.ff_norm.weight": "model-00007-of-00008.safetensors",
     "model.transformer.blocks.33.mlp.ff_out.weight": "model-00007-of-00008.safetensors",
     "model.transformer.blocks.33.mlp.ff_proj.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.33.self_attn.att_proj.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.33.self_attn.attn_out.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.33.self_attn.k_norm.weight": "model-00007-of-00008.safetensors",
-    "model.transformer.blocks.33.self_attn.q_norm.weight": "model-00007-of-00008.safetensors",
+    "model.transformer.blocks.33.self_attn.att_proj.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.33.self_attn.attn_out.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.33.self_attn.k_norm.weight": "model-00006-of-00008.safetensors",
+    "model.transformer.blocks.33.self_attn.q_norm.weight": "model-00006-of-00008.safetensors",
     "model.transformer.blocks.34.attn_norm.weight": "model-00007-of-00008.safetensors",
     "model.transformer.blocks.34.ff_norm.weight": "model-00007-of-00008.safetensors",
     "model.transformer.blocks.34.mlp.ff_out.weight": "model-00007-of-00008.safetensors",
@@ -283,37 +285,37 @@
     "model.transformer.blocks.5.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.5.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.6.attn_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.6.ff_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.6.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.6.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.6.ff_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.6.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.6.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.6.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.6.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.6.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.6.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
-    "model.transformer.blocks.7.attn_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.ff_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.7.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.attn_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.ff_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.8.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.9.attn_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.7.attn_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.ff_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.7.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.attn_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.ff_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.mlp.ff_out.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.mlp.ff_proj.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.8.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.9.attn_norm.weight": "model-00002-of-00008.safetensors",
     "model.transformer.blocks.9.ff_norm.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.9.mlp.ff_out.weight": "model-00003-of-00008.safetensors",
     "model.transformer.blocks.9.mlp.ff_proj.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.9.self_attn.att_proj.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.9.self_attn.attn_out.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.9.self_attn.k_norm.weight": "model-00003-of-00008.safetensors",
-    "model.transformer.blocks.9.self_attn.q_norm.weight": "model-00003-of-00008.safetensors",
+    "model.transformer.blocks.9.self_attn.att_proj.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.9.self_attn.attn_out.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.9.self_attn.k_norm.weight": "model-00002-of-00008.safetensors",
+    "model.transformer.blocks.9.self_attn.q_norm.weight": "model-00002-of-00008.safetensors",
     "model.transformer.ln_f.weight": "model-00007-of-00008.safetensors",
     "model.transformer.wte.embedding": "model-00001-of-00008.safetensors",
     "model.transformer.wte.new_embedding": "model-00001-of-00008.safetensors",
@@ -352,166 +354,166 @@
     "model.vit.transformer.resblocks.1.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.1.ffn_norm.bias": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.1.ffn_norm.weight": "model-00007-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.10.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.11.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.12.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.13.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.14.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.15.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.16.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.17.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.18.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.19.ffn_norm.weight": "model-00008-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.10.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.11.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.12.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.13.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.14.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.15.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.16.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.17.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.18.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.19.ffn_norm.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.2.attention.wk.bias": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.2.attention.wk.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.2.attention.wo.bias": "model-00007-of-00008.safetensors",
@@ -528,86 +530,86 @@
     "model.vit.transformer.resblocks.2.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.2.ffn_norm.bias": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.2.ffn_norm.weight": "model-00007-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.20.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.21.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.22.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.23.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.24.ffn_norm.weight": "model-00008-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.20.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.21.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.22.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.23.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.24.ffn_norm.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.3.attention.wk.bias": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.3.attention.wk.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.3.attention.wo.bias": "model-00007-of-00008.safetensors",
@@ -696,32 +698,30 @@
     "model.vit.transformer.resblocks.8.attention.wq.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.8.attention.wv.bias": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.8.attention.wv.weight": "model-00007-of-00008.safetensors",
-    "model.vit.transformer.resblocks.8.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.8.attention_norm.weight": "model-00008-of-00008.safetensors",
+    "model.vit.transformer.resblocks.8.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.8.attention_norm.weight": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.8.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
     "model.vit.transformer.resblocks.8.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
-    "model.vit.transformer.resblocks.8.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.8.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.8.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.8.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wk.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wk.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wo.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wo.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wq.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wq.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wv.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention.wv.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.attention_norm.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.feed_forward.w1.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.feed_forward.w1.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.feed_forward.w2.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.feed_forward.w2.weight": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.ffn_norm.bias": "model-00008-of-00008.safetensors",
-    "model.vit.transformer.resblocks.9.ffn_norm.weight": "model-00008-of-00008.safetensors",
-    "model.x_norm.weight": "model-00008-of-00008.safetensors",
-    "new_output_embeddings": "model-00001-of-00008.safetensors",
-    "output_embeddings": "model-00001-of-00008.safetensors"
+    "model.vit.transformer.resblocks.8.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.8.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.8.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.8.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wk.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wk.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wo.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wo.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wq.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wq.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wv.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention.wv.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.attention_norm.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.feed_forward.w1.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.feed_forward.w1.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.feed_forward.w2.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.feed_forward.w2.weight": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.ffn_norm.bias": "model-00007-of-00008.safetensors",
+    "model.vit.transformer.resblocks.9.ffn_norm.weight": "model-00007-of-00008.safetensors",
+    "model.x_norm.weight": "model-00007-of-00008.safetensors"
   }
 }

modeling_molmo_point.py

@@ -1307,9 +1307,10 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
             input_patch_ids = None
             can_point = False
 
-        x = self.transformer.wte(input_ids)
+        device = input_ids.device
+        x = self.transformer.wte(input_ids).to(device=device)
         batch_size, _, dim = x.shape
-        batch_idx = torch.arange(batch_size, device=self.device)
+        batch_idx = torch.arange(batch_size, device=device)
 
         # TODO update embeddings for patch/subpatch tokens
         vit_features_flat: Optional[torch.FloatTensor] = None
@@ -1326,7 +1327,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
             features = []
             for layer in self.vit_layers:
                 features.append(vit_image_features[layer])
-            vit_features = torch.cat(features, dim=-1)
+            vit_features = torch.cat(features, dim=-1).to(device=device)
             vit_feature_dim = vit_features.shape[-1]
 
             # Gather the features that should be pooled to build patch embeddings
@@ -1342,7 +1343,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
             vit_features_to_flat_mask = vit_features_mask.view(-1, token_pooling.shape[-1])[image_features_mask.view(-1)]
 
             # Finally apply the connector and add to input embeddings
-            image_features = self.connector(vit_features_flat, vit_features_to_flat_mask)
+            image_features = self.connector(vit_features_flat, vit_features_to_flat_mask).to(device=device)
             x = x.clone()
             x.view(-1, dim)[is_image_token.view(-1)] += image_features.view(-1, dim)
 
@@ -1350,7 +1351,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
             # embeddings
             image_token_indices = torch.cumsum(is_indexable_image_token, dim=-1) - 1
             image_pos_ids_flat = image_token_indices.view(-1)[is_image_token.view(-1)]
-            image_pos_ids = torch.zeros([batch_size, token_pooling.shape[1]], dtype=torch.long, device=self.device)
+            image_pos_ids = torch.zeros([batch_size, token_pooling.shape[1]], dtype=torch.long, device=device)
             image_pos_ids.view(-1)[image_features_mask.view(-1)] = image_pos_ids_flat
             max_image_pos_id = image_pos_ids_flat.max() + 1
         elif image_data is not None:
@@ -1374,7 +1375,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
                 assert last_predicted_patch_id is not None, "Patch should always be generated before a subpatch"
                 for_patches = (last_predicted_patch_id.view(batch_size) + image_token_offset)[input_subpatch_ids.view(batch_size) >= 0]
                 vit_features_to_embed = vit_features_flat[for_patches, input_subpatch_ids]
-                x.view(-1, dim)[is_subpatch.view(-1)] = self.build_vit_embedding(vit_features_to_embed).to(x.dtype)
+                x.view(-1, dim)[is_subpatch.view(-1)] = self.build_vit_embedding(vit_features_to_embed).to(device=device)
 
         # shape: (batch_size, seq_len, d_model)
         x = self.transformer.emb_drop(x)  # type: ignore
@@ -1438,7 +1439,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
 
         if images is not None or image_data is not None:
             if self.x_norm:
-                x_norm = self.x_norm(x)
+                x_norm = self.x_norm(x).to(device=device)
             elif self.config.norm_x:
                 x_norm = x / math.sqrt(dim)
             else:
@@ -1452,7 +1453,8 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
                 patch_k_flat = self.patch_k(x_norm.view(-1, dim)[is_image_token.view(-1)])
                 if self.patch_rotary is not None:
                     patch_k_flat = self.patch_rotary(patch_k_flat, image_pos_ids_flat)
-                patch_k = torch.zeros([batch_size, image_features_mask.shape[1], patch_k_flat.shape[-1]], dtype=x.dtype, device=self.device)
+                patch_k_flat = patch_k_flat.to(device=device)
+                patch_k = torch.zeros([batch_size, image_features_mask.shape[1], patch_k_flat.shape[-1]], dtype=x.dtype, device=device)
                 patch_k.view(-1, patch_k_flat.shape[-1])[image_features_mask.flatten()] = patch_k_flat.to(dtype=x.dtype)
 
                 patch_k_mask = image_features_mask.clone()
@@ -1460,14 +1462,14 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
                     is_indexable_image_token.view(-1)[is_image_token.view(-1)])
 
                 if self.config.no_more_points_class:
-                    patch_k = self.add_no_point_class_embed(patch_k)
+                    patch_k = self.add_no_point_class_embed(patch_k).to(device=device)
                     patch_k_mask = F.pad(patch_k_mask, (0, 1), value=True)
 
-                subpatch_k = self.subpatch_k(vit_features)
+                subpatch_k = self.subpatch_k(vit_features).to(device=device)
 
             # Predict patch locations
             if can_point:
-                image_q = self.patch_q(x_norm)
+                image_q = self.patch_q(x_norm).to(device=device)
                 if self.patch_rotary is not None and last_predicted_patch_id is not None:
                     rotate_by = image_pos_ids[batch_idx, last_predicted_patch_id]
                     rotate_by = torch.where(last_predicted_patch_id >= 0, rotate_by, 0)
@@ -1475,7 +1477,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
                     image_q = self.patch_rotary(
                         image_q.view(-1, image_q.shape[-1]),
                         torch.clamp(rotate_by, min=0),
-                    ).reshape(batch_size, -1, image_q.shape[-1])
+                    ).reshape(batch_size, -1, image_q.shape[-1]).to(device=device)
 
                 dots = torch.matmul(image_q, patch_k.transpose(1, 2))  # [batch, 1, num_images]
                 if self.config.norm_logits:
@@ -1487,7 +1489,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
             if can_point and torch.any(is_patch):
                 if x_norm.shape[1] != 1:
                     raise NotImplementedError()
-                subpatch_point_q = self.subpatch_q(x_norm.squeeze(1))
+                subpatch_point_q = self.subpatch_q(x_norm.squeeze(1)).to(device=device)
                 subpatch_k = subpatch_k[batch_idx, input_patch_ids.squeeze(1)]
                 subpatch_logits = torch.einsum("pd,pcd->pc", subpatch_point_q, subpatch_k)
                 if self.config.norm_logits:
@@ -1497,7 +1499,7 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
                 subpatch_logits = subpatch_logits[:, None, :]
 
             if can_point and torch.any(is_subpatch):
-                location_logits = self.subpatch_loc_k(x)
+                location_logits = self.subpatch_loc_k(x).to(device=device)
 
             if is_prefill:
                 num_image_tokens = is_image_token.sum(-1)
@@ -1534,6 +1536,17 @@ class MolmoPointModel(MolmoPointPreTrainedModel):
         )
 
 
+class ExtendedLmHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.output_embeddings = nn.Parameter(torch.zeros([config.vocab_size, config.hidden_size]))
+        self.new_output_embeddings = nn.Parameter(torch.zeros([128, config.hidden_size]))
+
+    def __call__(self, hidden_states, slice_indices=None):
+        lm_head = torch.concatenate([self.output_embeddings, self.new_output_embeddings], dim=0)
+        return F.linear(hidden_states[:, slice_indices, :], lm_head)
+
+
 class MolmoPointForConditionalGeneration(MolmoPointPreTrainedModel, GenerationMixin):
     _checkpoint_conversion_mapping = {}
     _tied_weights_keys = []  # Weights are not tied
@@ -1545,8 +1558,7 @@ class MolmoPointForConditionalGeneration(MolmoPointPreTrainedModel, GenerationMi
         super().__init__(config)
 
         self.model = MolmoPointModel(config)
-        self.output_embeddings = nn.Parameter(torch.zeros([config.vocab_size, config.hidden_size]))
-        self.new_output_embeddings = nn.Parameter(torch.zeros([128, config.hidden_size]))
+        self.lm_head = ExtendedLmHead(config)
         self.vocab_size = config.vocab_size
 
         # Initialize weights and apply final processing
@@ -1675,8 +1687,7 @@ class MolmoPointForConditionalGeneration(MolmoPointPreTrainedModel, GenerationMi
         hidden_states = outputs.last_hidden_state
         # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
         slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
-        lm_head = torch.concatenate([self.output_embeddings, self.new_output_embeddings], dim=0)
-        logits = F.linear(hidden_states[:, slice_indices, :], lm_head)
+        logits = self.lm_head(hidden_states, slice_indices=slice_indices)
 
         loss = None
         if labels is not None:
@@ -1697,6 +1708,7 @@ class MolmoPointForConditionalGeneration(MolmoPointPreTrainedModel, GenerationMi
         # process is hard to emulate in generation frameworks
         # Our hack here is to assume that, if we generate a TOKEN, we always select the argmax
         # patch. Then we can use PATCH_TOKEN scores as the argmax's patch scores
+        device = logits.device
         predicted_tokens = torch.argmax(logits[:, -1], dim=-1)
         patch_token_logits = torch.clone(logits[:, :, self.config.patch_token_id])
         logits[:, :, self.config.patch_token_id] = small_val
@@ -1705,8 +1717,8 @@ class MolmoPointForConditionalGeneration(MolmoPointPreTrainedModel, GenerationMi
         if outputs.patch_logits is not None:
             selected_patches = torch.argmax(outputs.patch_logits, -1)
             bs, seq, n_patches = outputs.patch_logits.shape
-            batch_idx = torch.arange(outputs.patch_logits.shape[0], device=self.device)
-            seq_ix = torch.arange(outputs.patch_logits.shape[1], device=self.device)
+            batch_idx = torch.arange(outputs.patch_logits.shape[0], device=device)
+            seq_ix = torch.arange(outputs.patch_logits.shape[1], device=device)
             argmax_patch_logits[batch_idx.view(-1, 1, 1), seq_ix.view(1, -1, 1), selected_patches] = patch_token_logits
 
         logits[:, :, self.config.subpatch_token_id] = small_val
@@ -1722,7 +1734,11 @@ class MolmoPointForConditionalGeneration(MolmoPointPreTrainedModel, GenerationMi
             location_logits = torch.full([bs, seq, 9], small_val, dtype=logits.dtype, device=logits.device)
 
         logits = torch.concatenate([
-            logits, argmax_patch_logits, subpatch_logits, location_logits], -1)
+            logits,
+            argmax_patch_logits,
+            subpatch_logits.to(device=device),
+            location_logits.to(device=device)
+        ], -1)
 
         return MolmoPointCausalLMOutputWithPast(
             loss=loss,