use bin instead of safetensors with max shard of 2GB

.gitattributes

@@ -33,5 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-model-00001-of-00002.safetensors filter=lfs diff=lfs merge=lfs -text
-model-00002-of-00002.safetensors filter=lfs diff=lfs merge=lfs -text
+tokenizer.model filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00003-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00004-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00006-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00007-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00008-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00001-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00002-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00005-of-00008.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin.index.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -27,8 +27,8 @@
   },
   "rope_theta": 1000000,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.36.2",
+  "torch_dtype": "float16",
+  "transformers_version": "4.33.2",
   "use_cache": true,
   "vocab_size": 92544
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 1,
   "eos_token_id": 2,
   "pad_token_id": 2,
-  "transformers_version": "4.36.2"
+  "transformers_version": "4.33.2"
 }

model.safetensors.index.json

@@ -1,234 +0,0 @@
-{
-  "metadata": {
-    "total_size": 15475417088
-  },
-  "weight_map": {
-    "model.layers.0.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.21.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.21.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.3.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.30.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.attention.wo.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.attention.wqkv.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.attention_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.feed_forward.w1.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.feed_forward.w2.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.ffn_norm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.4.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.wo.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.wqkv.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention_norm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.feed_forward.w1.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.feed_forward.w2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.ffn_norm.weight": "model-00001-of-00002.safetensors",
-    "model.norm.weight": "model-00002-of-00002.safetensors",
-    "model.tok_embeddings.weight": "model-00001-of-00002.safetensors",
-    "output.weight": "model-00002-of-00002.safetensors"
-  }
-}

modeling_internlm2.py

@@ -133,7 +133,7 @@ class InternLM2RotaryEmbedding(nn.Module):
     def forward(self, x, seq_len=None):
         # x: [bs, num_attention_heads, seq_len, head_size]
         if seq_len > self.max_seq_len_cached:
-            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32)
 
         return (
             self.cos_cached[:seq_len].to(dtype=x.dtype),
@@ -196,20 +196,10 @@ def rotate_half(x):
 
 
 def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
-    # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
-    cos = cos.squeeze(1).squeeze(0)  # [seq_len, dim]
-    sin = sin.squeeze(1).squeeze(0)  # [seq_len, dim]
-    cos = cos.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
-    sin = sin.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
-    if q.size(2) == 1:
-        q_embed = (q * cos[:, :, -1, :]) + (rotate_half(q) * sin[:, :, -1, :])
-    else:
-        q_embed = (q * cos) + (rotate_half(q) * sin)
-
-    if k.size(2) == 1:
-        k_embed = (k * cos[:, :, -1, :]) + (rotate_half(k) * sin[:, :, -1, :])
-    else:
-        k_embed = (k * cos) + (rotate_half(k) * sin)
+    cos = cos[position_ids].unsqueeze(1)
+    sin = sin[position_ids].unsqueeze(1)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
 
     return q_embed, k_embed
 
@@ -289,8 +279,15 @@ class InternLM2Attention(nn.Module):
                     base=self.config.rope_theta,
                     scaling_factor=scaling_factor
                 )
+            elif scaling_type == "linear":
+                self.rotary_emb = InternLM2LinearScalingRotaryEmbedding(
+                    self.head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    base=self.config.rope_theta,
+                    scaling_factor=scaling_factor
+                )
             else:
-                raise ValueError("Currently we only support rotary embedding's type being 'dynamic'.")
+                raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.")
         return self.rotary_emb
 
     def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
@@ -1032,7 +1029,6 @@ class InternLM2ForCausalLM(InternLM2PreTrainedModel):
         for record in history:
             prompt += f"""[UNUSED_TOKEN_146]user\n{record[0]}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n{record[1]}[UNUSED_TOKEN_145]\n"""
         prompt += f"""[UNUSED_TOKEN_146]user\n{query}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n"""
-        print(prompt)
         return tokenizer([prompt], return_tensors="pt")
 
     @torch.no_grad()
@@ -1268,5 +1264,5 @@ class InternLM2ForSequenceClassification(InternLM2PreTrainedModel):
             logits=pooled_logits,
             past_key_values=transformer_outputs.past_key_values,
             hidden_states=transformer_outputs.hidden_states,
-            attentions=transformer_outputs,
+            attentions=transformer_outputs.attentions,
         )

model-00001-of-00002.safetensors → pytorch_model-00001-of-00008.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:022c4bc3f8b1add6867d5333c6a4c26b8d4dd2e6a8f7accc6d15cc749a790db3
-size 9969173040
+oid sha256:fdd03c1ea040b17ce992eb303d0824f5694932902d0c3fa57b5a8d1df2dc8082
+size 1949342053

model-00002-of-00002.safetensors → pytorch_model-00002-of-00008.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9d6ddfb8d8371c2751b11f3c4477431d5ac16e65cf5d1e6cdb4775d899b82b9
-size 5506269856
+oid sha256:377d66327bcc2dc4b3e5f4e90f24b428c55832ece07194990248647ea65c2db5
+size 1946249825

pytorch_model-00003-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98aa01b7ffd7901699b4588b5e6b8f753e66c1874f8b855c36b23de2c95ae9d8
+size 1979786923

pytorch_model-00004-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9362abdb2afd077e579a7860ad570769255c2d64af2b0831413578c77a44d779
+size 1946249889

pytorch_model-00005-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56761cf09eb95e676bce5c1dc53fd4f8c317775d146fe2313dcb7b81c48cfa39
+size 1979786987

pytorch_model-00006-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a825cc1e85ffc038efaed5a06ae398f424916bdd768bcf62d7fb4bc552d71ef
+size 1946249889

pytorch_model-00007-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:224105eee8f19409994aea890cc6cc9da16ef4278ad831fdd4e9fcf86f925256
+size 1979786987

pytorch_model-00008-of-00008.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ed276d23e8a4ab9ec5bafe491cb7147b4b51232a66ce0b9f304bc8a5430d82f
+size 1748039973

pytorch_model.bin.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8a1efb6998624330a0564f9bba63eb8ccae0ad54a6d0176c64f2eb30721f2b5
+size 18179

tokenizer_config.json

@@ -1,30 +1,4 @@
 {
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
   "auto_map": {
     "AutoTokenizer": [
       "tokenization_internlm.InternLMTokenizer",