End of training

README.md

@@ -17,7 +17,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) on the openwebtext dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.8068
+- Loss: 2.7862
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- training_steps: 4
+- training_steps: 2
 
 ### Training results
 

config.json

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.1",
   "architectures": [
-    "MistralForCausalLM"
+    "SparseMistral"
   ],
   "bos_token_id": 1,
   "eos_token_id": 2,
@@ -9,17 +9,21 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 32768,
+  "max_position_embeddings": 131072,
   "model_type": "mistral",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "rms_norm_eps": 1e-05,
+  "regularization_coefficient": 10,
+  "regularization_type": "L1 Regularization",
+  "rms_norm_eps": 1e-06,
   "rope_theta": 10000.0,
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.35.2",
   "use_cache": true,
+  "use_sparse_model": false,
+  "use_sparse_regularization": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d09a0990c3abed9a9cddeb6d07bb7387f66f8467a5d59e043a5368fb05d53062
+oid sha256:ce255e1c678bab67d40e0e7470e0bff12a4cabea3cac6dcb8f41c2adf0b57c9a
 size 4943163992

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b5938632792265c598d6a194efb8494a5aca5d723162b318c71e6b2d3b56511
+oid sha256:68e21ccc1b9703ae7a6ec4264f248cf3f8f9227b24723d96438e16d265d5443b
 size 4999821144

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b19f9ffda065181ed378b64943ea0cd914f9a62cbfe33e9551e0d2a012bcecbb
+oid sha256:3f99e43cac8aa500195dae0c7f6fe4fdc6c1a5353ee99b1a63ab6b5385852461
 size 4540517840

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29bf7b0f483d11f7eebbccc253ff87fcc38a3c6c23e898925cb94341a5dcbc28
+oid sha256:0edd9a9aea652105a2d8eed2ea59d50affc6d9013c74ab173064adef41bfe0fb
 size 6008