End of training

README.md

@@ -17,7 +17,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) on the openwebtext dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.9177
+- Loss: 1.8068
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- training_steps: 2
+- training_steps: 4
 
 ### Training results
 

config.json

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.1",
   "architectures": [
-    "SparseMistral"
+    "MistralForCausalLM"
   ],
   "bos_token_id": 1,
   "eos_token_id": 2,
@@ -9,21 +9,17 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 131072,
+  "max_position_embeddings": 32768,
   "model_type": "mistral",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "regularization_coefficient": 10,
-  "regularization_type": "L1 Regularization",
-  "rms_norm_eps": 1e-06,
+  "rms_norm_eps": 1e-05,
   "rope_theta": 10000.0,
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.35.2",
   "use_cache": true,
-  "use_sparse_model": false,
-  "use_sparse_regularization": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36c02e05ed195fcba3ae7f5260df1a5f27a70f69d59fef18c8743da6f2a6873f
+oid sha256:d09a0990c3abed9a9cddeb6d07bb7387f66f8467a5d59e043a5368fb05d53062
 size 4943163992

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63630fde5d276c113df7f4a90006478c824032ef54d46c222e7afaf499183468
+oid sha256:5b5938632792265c598d6a194efb8494a5aca5d723162b318c71e6b2d3b56511
 size 4999821144

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e40f0e62ec2a0ffbd5979bd013bc81fdaf427c8ba293cab39d4305fc399573e2
+oid sha256:b19f9ffda065181ed378b64943ea0cd914f9a62cbfe33e9551e0d2a012bcecbb
 size 4540517840

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0edd9a9aea652105a2d8eed2ea59d50affc6d9013c74ab173064adef41bfe0fb
+oid sha256:29bf7b0f483d11f7eebbccc253ff87fcc38a3c6c23e898925cb94341a5dcbc28
 size 6008