v2

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f06ab051b83533d3fa8f4e7fc2dfbe3a94cbe642d4c47f19cedcfdee8816e43d
+oid sha256:83ea039a5fd99654448604bb37a8b98e48d05101e4b070b01a30bac5ad38c785
 size 1550795272

model.safetensors.index.json

@@ -0,0 +1,307 @@
+{
+  "weight_map": {
+    "lm_head.weight": "model.safetensors",
+    "lm_head_scale": "model.safetensors",
+    "model.embed_tokens.weight": "model.safetensors",
+    "model.layers.0.attn_branch_scale": "model.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.0.mlp.gate_scale": "model.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.0.mlp.up_scale": "model.safetensors",
+    "model.layers.0.mlp_branch_scale": "model.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.0.self_attn.qk_scale": "model.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.1.attn_branch_scale": "model.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.1.mlp.gate_scale": "model.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.1.mlp.up_scale": "model.safetensors",
+    "model.layers.1.mlp_branch_scale": "model.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.1.self_attn.qk_scale": "model.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.10.attn_branch_scale": "model.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.10.mlp.gate_scale": "model.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.10.mlp.up_scale": "model.safetensors",
+    "model.layers.10.mlp_branch_scale": "model.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.10.self_attn.qk_scale": "model.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.11.attn_branch_scale": "model.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.11.mlp.gate_scale": "model.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.11.mlp.up_scale": "model.safetensors",
+    "model.layers.11.mlp_branch_scale": "model.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.11.self_attn.qk_scale": "model.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.12.attn_branch_scale": "model.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.12.mlp.gate_scale": "model.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.12.mlp.up_scale": "model.safetensors",
+    "model.layers.12.mlp_branch_scale": "model.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.12.self_attn.qk_scale": "model.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.13.attn_branch_scale": "model.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.13.mlp.gate_scale": "model.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.13.mlp.up_scale": "model.safetensors",
+    "model.layers.13.mlp_branch_scale": "model.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.13.self_attn.qk_scale": "model.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.14.attn_branch_scale": "model.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.14.mlp.gate_scale": "model.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.14.mlp.up_scale": "model.safetensors",
+    "model.layers.14.mlp_branch_scale": "model.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.14.self_attn.qk_scale": "model.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.15.attn_branch_scale": "model.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.15.mlp.gate_scale": "model.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.15.mlp.up_scale": "model.safetensors",
+    "model.layers.15.mlp_branch_scale": "model.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.15.self_attn.qk_scale": "model.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.16.attn_branch_scale": "model.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.16.mlp.gate_scale": "model.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.16.mlp.up_scale": "model.safetensors",
+    "model.layers.16.mlp_branch_scale": "model.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.16.self_attn.qk_scale": "model.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.17.attn_branch_scale": "model.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.17.mlp.gate_scale": "model.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.17.mlp.up_scale": "model.safetensors",
+    "model.layers.17.mlp_branch_scale": "model.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.17.self_attn.qk_scale": "model.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.18.attn_branch_scale": "model.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.18.mlp.gate_scale": "model.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.18.mlp.up_scale": "model.safetensors",
+    "model.layers.18.mlp_branch_scale": "model.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.18.self_attn.qk_scale": "model.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.19.attn_branch_scale": "model.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.19.mlp.gate_scale": "model.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.19.mlp.up_scale": "model.safetensors",
+    "model.layers.19.mlp_branch_scale": "model.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.19.self_attn.qk_scale": "model.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.2.attn_branch_scale": "model.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.2.mlp.gate_scale": "model.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.2.mlp.up_scale": "model.safetensors",
+    "model.layers.2.mlp_branch_scale": "model.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.2.self_attn.qk_scale": "model.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.20.attn_branch_scale": "model.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.20.mlp.gate_scale": "model.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.20.mlp.up_scale": "model.safetensors",
+    "model.layers.20.mlp_branch_scale": "model.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.20.self_attn.qk_scale": "model.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.21.attn_branch_scale": "model.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.21.mlp.gate_scale": "model.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.21.mlp.up_scale": "model.safetensors",
+    "model.layers.21.mlp_branch_scale": "model.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.21.self_attn.qk_scale": "model.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.22.attn_branch_scale": "model.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.22.mlp.gate_scale": "model.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.22.mlp.up_scale": "model.safetensors",
+    "model.layers.22.mlp_branch_scale": "model.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.22.self_attn.qk_scale": "model.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.23.attn_branch_scale": "model.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.23.mlp.gate_scale": "model.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.23.mlp.up_scale": "model.safetensors",
+    "model.layers.23.mlp_branch_scale": "model.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.23.self_attn.qk_scale": "model.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.24.attn_branch_scale": "model.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.24.mlp.gate_scale": "model.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.24.mlp.up_scale": "model.safetensors",
+    "model.layers.24.mlp_branch_scale": "model.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.24.self_attn.qk_scale": "model.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.25.attn_branch_scale": "model.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.25.mlp.gate_scale": "model.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.25.mlp.up_scale": "model.safetensors",
+    "model.layers.25.mlp_branch_scale": "model.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.25.self_attn.qk_scale": "model.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.26.attn_branch_scale": "model.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.26.mlp.gate_scale": "model.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.26.mlp.up_scale": "model.safetensors",
+    "model.layers.26.mlp_branch_scale": "model.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.26.self_attn.qk_scale": "model.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.27.attn_branch_scale": "model.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.27.mlp.gate_scale": "model.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.27.mlp.up_scale": "model.safetensors",
+    "model.layers.27.mlp_branch_scale": "model.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.27.self_attn.qk_scale": "model.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.28.attn_branch_scale": "model.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.28.mlp.gate_scale": "model.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.28.mlp.up_scale": "model.safetensors",
+    "model.layers.28.mlp_branch_scale": "model.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.28.self_attn.qk_scale": "model.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.29.attn_branch_scale": "model.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.29.mlp.gate_scale": "model.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.29.mlp.up_scale": "model.safetensors",
+    "model.layers.29.mlp_branch_scale": "model.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.29.self_attn.qk_scale": "model.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.3.attn_branch_scale": "model.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.3.mlp.gate_scale": "model.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.3.mlp.up_scale": "model.safetensors",
+    "model.layers.3.mlp_branch_scale": "model.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.3.self_attn.qk_scale": "model.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.4.attn_branch_scale": "model.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.4.mlp.gate_scale": "model.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.4.mlp.up_scale": "model.safetensors",
+    "model.layers.4.mlp_branch_scale": "model.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.4.self_attn.qk_scale": "model.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.5.attn_branch_scale": "model.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.5.mlp.gate_scale": "model.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.5.mlp.up_scale": "model.safetensors",
+    "model.layers.5.mlp_branch_scale": "model.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.5.self_attn.qk_scale": "model.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.6.attn_branch_scale": "model.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.6.mlp.gate_scale": "model.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.6.mlp.up_scale": "model.safetensors",
+    "model.layers.6.mlp_branch_scale": "model.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.6.self_attn.qk_scale": "model.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.7.attn_branch_scale": "model.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.7.mlp.gate_scale": "model.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.7.mlp.up_scale": "model.safetensors",
+    "model.layers.7.mlp_branch_scale": "model.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.7.self_attn.qk_scale": "model.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.8.attn_branch_scale": "model.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.8.mlp.gate_scale": "model.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.8.mlp.up_scale": "model.safetensors",
+    "model.layers.8.mlp_branch_scale": "model.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.8.self_attn.qk_scale": "model.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model.safetensors",
+    "model.layers.9.attn_branch_scale": "model.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model.safetensors",
+    "model.layers.9.mlp.gate_scale": "model.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model.safetensors",
+    "model.layers.9.mlp.up_scale": "model.safetensors",
+    "model.layers.9.mlp_branch_scale": "model.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model.safetensors",
+    "model.layers.9.self_attn.qk_scale": "model.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model.safetensors"
+  }
+}

modeling_ngpt.py

@@ -1229,7 +1229,7 @@ class nGPTForCausalLM(LlamaPreTrainedModel, GenerationMixin):
         else:
             # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
             logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
-            logits = logits * self.lm_head_scale
+            logits = logits * self.lm_head_scale.to(logits.device)
         # print(f"after lm_head logits: {logits}")
         
         # print(f"final logits: {logits.shape} {logits.squeeze(0)}")

tokenizer_config.json

@@ -33,11 +33,11 @@
   "eos_token": "</s>",
   "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
+  "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}",
   "pad_token": null,
   "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
-  "chat_template": "{{ '### Instruction:\n' }}{% for message in messages %}{{message['content'] + ' '}}{% endfor %}{% if add_generation_prompt %}{{ '\n\n### Response:\n' }}{% endif %}",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }