Model save

README.md

@@ -4,7 +4,6 @@ library_name: transformers
 model_name: llama_check_tuned
 tags:
 - generated_from_trainer
-- alignment-handbook
 - trl
 - sft
 licence: license
@@ -28,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/vedaantj/ft-llm/runs/ierr3rjg)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/vedaantj/ft-llm/runs/z2e7k8ju)
 
 This model was trained with SFT.
 
@@ -36,8 +35,8 @@ This model was trained with SFT.
 
 - TRL: 0.12.2
 - Transformers: 4.46.3
-- Pytorch: 2.4.1
-- Datasets: 3.1.0
+- Pytorch: 2.4.1+cu124
+- Datasets: 3.2.0
 - Tokenizers: 0.20.3
 
 ## Citations

all_results.json

@@ -6,9 +6,9 @@
     "eval_samples_per_second": 58.378,
     "eval_steps_per_second": 11.676,
     "total_flos": 4849620932886528.0,
-    "train_loss": 0.6959351973579032,
-    "train_runtime": 31.7849,
+    "train_loss": 0.7576449837215478,
+    "train_runtime": 31.485,
     "train_samples": 10,
-    "train_samples_per_second": 12.585,
-    "train_steps_per_second": 1.258
+    "train_samples_per_second": 12.704,
+    "train_steps_per_second": 1.27
 }

config.json

@@ -22,7 +22,7 @@
   "num_attention_heads": 24,
   "num_hidden_layers": 28,
   "num_key_value_heads": 8,
-  "pad_token_id": 128009,
+  "pad_token_id": 128002,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
@@ -36,6 +36,6 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.46.3",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 128256
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1c5a4c3b8aa8f79f141452fcaf4d59840224c3af6c6b366b00f97903fe6c4e0
+oid sha256:080685cef97854956a346ddb907ae1abce575d0a719f969f996c1dc3f387d4d6
 size 4965799096

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dba518352df14b4438c4b2b16ddd67fa6fb811472fd5fb67395889f198be0c71
+oid sha256:421cc2c9708aead6940c83b813c7aee1bf6fd02ca11e5b3efec1cb1e8d0542a5
 size 2247734992

special_tokens_map.json

@@ -13,5 +13,11 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|eot_id|>"
+  "pad_token": {
+    "content": "<|reserved_special_token_0|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
-size 17209920
+oid sha256:6fca926fdfedcb8fd225765fb3911e0ab5de0390ec7242a46dff829e8604f723
+size 17210020

tokenizer_config.json

@@ -2058,6 +2058,6 @@
     "attention_mask"
   ],
   "model_max_length": 2048,
-  "pad_token": "<|eot_id|>",
+  "pad_token": "<|reserved_special_token_0|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 40.0,
     "total_flos": 4849620932886528.0,
-    "train_loss": 0.6959351973579032,
-    "train_runtime": 31.7849,
+    "train_loss": 0.7576449837215478,
+    "train_runtime": 31.485,
     "train_samples": 10,
-    "train_samples_per_second": 12.585,
-    "train_steps_per_second": 1.258
+    "train_samples_per_second": 12.704,
+    "train_steps_per_second": 1.27
 }

trainer_state.json

@@ -11,610 +11,610 @@
     {
       "epoch": 1.0,
       "learning_rate": 2.5e-05,
-      "loss": 1.1447,
+      "loss": 1.1411,
       "step": 1
     },
     {
       "epoch": 1.0,
-      "eval_loss": 1.1312110424041748,
-      "eval_runtime": 0.1567,
-      "eval_samples_per_second": 63.798,
-      "eval_steps_per_second": 12.76,
+      "eval_loss": 1.1278259754180908,
+      "eval_runtime": 0.1561,
+      "eval_samples_per_second": 64.049,
+      "eval_steps_per_second": 12.81,
       "step": 1
     },
     {
       "epoch": 2.0,
-      "grad_norm": 9.785662651062012,
+      "grad_norm": 9.757247924804688,
       "learning_rate": 5e-05,
-      "loss": 1.1235,
+      "loss": 1.1201,
       "step": 2
     },
     {
       "epoch": 2.0,
-      "eval_loss": 0.7337616682052612,
-      "eval_runtime": 0.156,
-      "eval_samples_per_second": 64.086,
-      "eval_steps_per_second": 12.817,
+      "eval_loss": 0.7308214902877808,
+      "eval_runtime": 0.157,
+      "eval_samples_per_second": 63.676,
+      "eval_steps_per_second": 12.735,
       "step": 2
     },
     {
       "epoch": 3.0,
-      "grad_norm": 9.785662651062012,
+      "grad_norm": 9.757247924804688,
       "learning_rate": 7.500000000000001e-05,
-      "loss": 0.743,
+      "loss": 0.7406,
       "step": 3
     },
     {
       "epoch": 3.0,
-      "eval_loss": 0.7337616682052612,
-      "eval_runtime": 0.1556,
-      "eval_samples_per_second": 64.283,
-      "eval_steps_per_second": 12.857,
+      "eval_loss": 0.7308214902877808,
+      "eval_runtime": 0.1559,
+      "eval_samples_per_second": 64.133,
+      "eval_steps_per_second": 12.827,
       "step": 3
     },
     {
       "epoch": 4.0,
-      "grad_norm": 6.636180877685547,
+      "grad_norm": 6.656477451324463,
       "learning_rate": 0.0001,
-      "loss": 0.7504,
+      "loss": 0.7483,
       "step": 4
     },
     {
       "epoch": 4.0,
-      "eval_loss": 1.1203209161758423,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.304,
-      "eval_steps_per_second": 12.861,
+      "eval_loss": 1.1192941665649414,
+      "eval_runtime": 0.1548,
+      "eval_samples_per_second": 64.593,
+      "eval_steps_per_second": 12.919,
       "step": 4
     },
     {
       "epoch": 5.0,
-      "grad_norm": 6.636180877685547,
+      "grad_norm": 6.656477451324463,
       "learning_rate": 0.000125,
-      "loss": 1.1145,
+      "loss": 1.1127,
       "step": 5
     },
     {
       "epoch": 5.0,
-      "eval_loss": 1.1203209161758423,
-      "eval_runtime": 0.1562,
-      "eval_samples_per_second": 64.004,
-      "eval_steps_per_second": 12.801,
+      "eval_loss": 1.1192941665649414,
+      "eval_runtime": 0.1561,
+      "eval_samples_per_second": 64.065,
+      "eval_steps_per_second": 12.813,
       "step": 5
     },
     {
       "epoch": 6.0,
-      "grad_norm": 18.654922485351562,
+      "grad_norm": 18.55446434020996,
       "learning_rate": 0.00015000000000000001,
-      "loss": 1.0779,
+      "loss": 1.0754,
       "step": 6
     },
     {
       "epoch": 6.0,
-      "eval_loss": 0.9797636866569519,
-      "eval_runtime": 0.1554,
-      "eval_samples_per_second": 64.341,
-      "eval_steps_per_second": 12.868,
+      "eval_loss": 0.9689571261405945,
+      "eval_runtime": 0.1555,
+      "eval_samples_per_second": 64.312,
+      "eval_steps_per_second": 12.862,
       "step": 6
     },
     {
       "epoch": 7.0,
-      "grad_norm": 18.654922485351562,
+      "grad_norm": 18.55446434020996,
       "learning_rate": 0.000175,
-      "loss": 0.9815,
+      "loss": 0.9708,
       "step": 7
     },
     {
       "epoch": 7.0,
-      "eval_loss": 0.9797636866569519,
-      "eval_runtime": 0.1551,
-      "eval_samples_per_second": 64.482,
-      "eval_steps_per_second": 12.896,
+      "eval_loss": 0.9689571261405945,
+      "eval_runtime": 0.1553,
+      "eval_samples_per_second": 64.383,
+      "eval_steps_per_second": 12.877,
       "step": 7
     },
     {
       "epoch": 8.0,
-      "grad_norm": 13.476932525634766,
+      "grad_norm": 13.187333106994629,
       "learning_rate": 0.0002,
-      "loss": 1.0471,
+      "loss": 1.0331,
       "step": 8
     },
     {
       "epoch": 8.0,
-      "eval_loss": 5.148657321929932,
+      "eval_loss": 5.132155418395996,
       "eval_runtime": 0.156,
-      "eval_samples_per_second": 64.108,
-      "eval_steps_per_second": 12.822,
+      "eval_samples_per_second": 64.107,
+      "eval_steps_per_second": 12.821,
       "step": 8
     },
     {
       "epoch": 9.0,
-      "grad_norm": 13.476932525634766,
+      "grad_norm": 13.187333106994629,
       "learning_rate": 0.0001995184726672197,
-      "loss": 5.1291,
+      "loss": 5.0832,
       "step": 9
     },
     {
       "epoch": 9.0,
-      "eval_loss": 5.148657321929932,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.323,
-      "eval_steps_per_second": 12.865,
+      "eval_loss": 5.132155418395996,
+      "eval_runtime": 0.1552,
+      "eval_samples_per_second": 64.447,
+      "eval_steps_per_second": 12.889,
       "step": 9
     },
     {
       "epoch": 10.0,
-      "grad_norm": 141.67774963378906,
+      "grad_norm": 140.08033752441406,
       "learning_rate": 0.00019807852804032305,
-      "loss": 5.2219,
+      "loss": 5.1665,
       "step": 10
     },
     {
       "epoch": 10.0,
-      "eval_loss": 2.3491921424865723,
-      "eval_runtime": 0.1552,
-      "eval_samples_per_second": 64.421,
-      "eval_steps_per_second": 12.884,
+      "eval_loss": 3.2012367248535156,
+      "eval_runtime": 0.1572,
+      "eval_samples_per_second": 63.616,
+      "eval_steps_per_second": 12.723,
       "step": 10
     },
     {
       "epoch": 11.0,
-      "grad_norm": 141.67774963378906,
+      "grad_norm": 140.08033752441406,
       "learning_rate": 0.0001956940335732209,
-      "loss": 2.4209,
+      "loss": 3.2953,
       "step": 11
     },
     {
       "epoch": 11.0,
-      "eval_loss": 2.3491921424865723,
-      "eval_runtime": 0.156,
-      "eval_samples_per_second": 64.119,
-      "eval_steps_per_second": 12.824,
+      "eval_loss": 3.2012367248535156,
+      "eval_runtime": 0.1557,
+      "eval_samples_per_second": 64.221,
+      "eval_steps_per_second": 12.844,
       "step": 11
     },
     {
       "epoch": 12.0,
-      "grad_norm": 95.97136688232422,
+      "grad_norm": 122.30906677246094,
       "learning_rate": 0.0001923879532511287,
-      "loss": 2.4686,
+      "loss": 3.3383,
       "step": 12
     },
     {
       "epoch": 12.0,
-      "eval_loss": 0.9826704859733582,
-      "eval_runtime": 0.1558,
-      "eval_samples_per_second": 64.17,
-      "eval_steps_per_second": 12.834,
+      "eval_loss": 1.2309796810150146,
+      "eval_runtime": 0.1559,
+      "eval_samples_per_second": 64.128,
+      "eval_steps_per_second": 12.826,
       "step": 12
     },
     {
       "epoch": 13.0,
-      "grad_norm": 95.97136688232422,
+      "grad_norm": 122.30906677246094,
       "learning_rate": 0.0001881921264348355,
-      "loss": 0.986,
+      "loss": 1.2275,
       "step": 13
     },
     {
       "epoch": 13.0,
-      "eval_loss": 0.9826704859733582,
-      "eval_runtime": 0.1552,
-      "eval_samples_per_second": 64.426,
-      "eval_steps_per_second": 12.885,
+      "eval_loss": 1.2309796810150146,
+      "eval_runtime": 0.1566,
+      "eval_samples_per_second": 63.87,
+      "eval_steps_per_second": 12.774,
       "step": 13
     },
     {
       "epoch": 14.0,
-      "grad_norm": 7.2887864112854,
+      "grad_norm": 9.766098022460938,
       "learning_rate": 0.00018314696123025454,
-      "loss": 0.9912,
+      "loss": 1.2398,
       "step": 14
     },
     {
       "epoch": 14.0,
-      "eval_loss": 0.7040748596191406,
-      "eval_runtime": 0.1558,
-      "eval_samples_per_second": 64.185,
-      "eval_steps_per_second": 12.837,
+      "eval_loss": 0.7279261946678162,
+      "eval_runtime": 0.1553,
+      "eval_samples_per_second": 64.403,
+      "eval_steps_per_second": 12.881,
       "step": 14
     },
     {
       "epoch": 15.0,
-      "grad_norm": 7.2887864112854,
+      "grad_norm": 9.766098022460938,
       "learning_rate": 0.0001773010453362737,
-      "loss": 0.7151,
+      "loss": 0.7426,
       "step": 15
     },
     {
       "epoch": 15.0,
-      "eval_loss": 0.7040748596191406,
-      "eval_runtime": 0.1556,
-      "eval_samples_per_second": 64.283,
-      "eval_steps_per_second": 12.857,
+      "eval_loss": 0.7279261946678162,
+      "eval_runtime": 0.1568,
+      "eval_samples_per_second": 63.763,
+      "eval_steps_per_second": 12.753,
       "step": 15
     },
     {
       "epoch": 16.0,
-      "grad_norm": 7.857557773590088,
+      "grad_norm": 7.932348728179932,
       "learning_rate": 0.00017071067811865476,
-      "loss": 0.7103,
+      "loss": 0.7375,
       "step": 16
     },
     {
       "epoch": 16.0,
-      "eval_loss": 0.27107304334640503,
-      "eval_runtime": 0.1552,
-      "eval_samples_per_second": 64.413,
-      "eval_steps_per_second": 12.883,
+      "eval_loss": 0.4009631276130676,
+      "eval_runtime": 0.1564,
+      "eval_samples_per_second": 63.956,
+      "eval_steps_per_second": 12.791,
       "step": 16
     },
     {
       "epoch": 17.0,
-      "grad_norm": 7.857557773590088,
+      "grad_norm": 7.932348728179932,
       "learning_rate": 0.00016343932841636456,
-      "loss": 0.2735,
+      "loss": 0.4055,
       "step": 17
     },
     {
       "epoch": 17.0,
-      "eval_loss": 0.27107304334640503,
-      "eval_runtime": 0.1552,
-      "eval_samples_per_second": 64.433,
-      "eval_steps_per_second": 12.887,
+      "eval_loss": 0.4009631276130676,
+      "eval_runtime": 0.1554,
+      "eval_samples_per_second": 64.342,
+      "eval_steps_per_second": 12.868,
       "step": 17
     },
     {
       "epoch": 18.0,
-      "grad_norm": 3.201599597930908,
+      "grad_norm": 6.0723419189453125,
       "learning_rate": 0.00015555702330196023,
-      "loss": 0.2697,
+      "loss": 0.3971,
       "step": 18
     },
     {
       "epoch": 18.0,
-      "eval_loss": 0.15566985309123993,
-      "eval_runtime": 0.1556,
-      "eval_samples_per_second": 64.252,
-      "eval_steps_per_second": 12.85,
+      "eval_loss": 0.15298452973365784,
+      "eval_runtime": 0.1557,
+      "eval_samples_per_second": 64.239,
+      "eval_steps_per_second": 12.848,
       "step": 18
     },
     {
       "epoch": 19.0,
-      "grad_norm": 3.201599597930908,
+      "grad_norm": 6.0723419189453125,
       "learning_rate": 0.0001471396736825998,
-      "loss": 0.1503,
+      "loss": 0.1623,
       "step": 19
     },
     {
       "epoch": 19.0,
-      "eval_loss": 0.15566985309123993,
-      "eval_runtime": 0.1647,
-      "eval_samples_per_second": 60.711,
-      "eval_steps_per_second": 12.142,
+      "eval_loss": 0.15298452973365784,
+      "eval_runtime": 0.1553,
+      "eval_samples_per_second": 64.4,
+      "eval_steps_per_second": 12.88,
       "step": 19
     },
     {
       "epoch": 20.0,
-      "grad_norm": 2.2040343284606934,
+      "grad_norm": 2.892711639404297,
       "learning_rate": 0.000138268343236509,
-      "loss": 0.1456,
+      "loss": 0.1554,
       "step": 20
     },
     {
       "epoch": 20.0,
-      "eval_loss": 0.10551667213439941,
-      "eval_runtime": 0.1561,
-      "eval_samples_per_second": 64.064,
-      "eval_steps_per_second": 12.813,
+      "eval_loss": 0.09138597548007965,
+      "eval_runtime": 0.1562,
+      "eval_samples_per_second": 64.038,
+      "eval_steps_per_second": 12.808,
       "step": 20
     },
     {
       "epoch": 21.0,
-      "grad_norm": 2.2040343284606934,
+      "grad_norm": 2.892711639404297,
       "learning_rate": 0.00012902846772544624,
-      "loss": 0.1046,
+      "loss": 0.0876,
       "step": 21
     },
     {
       "epoch": 21.0,
-      "eval_loss": 0.10551667213439941,
-      "eval_runtime": 0.1585,
-      "eval_samples_per_second": 63.083,
-      "eval_steps_per_second": 12.617,
+      "eval_loss": 0.09138597548007965,
+      "eval_runtime": 0.1555,
+      "eval_samples_per_second": 64.296,
+      "eval_steps_per_second": 12.859,
       "step": 21
     },
     {
       "epoch": 22.0,
-      "grad_norm": 1.8842490911483765,
+      "grad_norm": 1.7072333097457886,
       "learning_rate": 0.00011950903220161285,
-      "loss": 0.1046,
+      "loss": 0.0933,
       "step": 22
     },
     {
       "epoch": 22.0,
-      "eval_loss": 0.03636971116065979,
-      "eval_runtime": 0.1553,
-      "eval_samples_per_second": 64.41,
-      "eval_steps_per_second": 12.882,
+      "eval_loss": 0.057169489562511444,
+      "eval_runtime": 0.1554,
+      "eval_samples_per_second": 64.345,
+      "eval_steps_per_second": 12.869,
       "step": 22
     },
     {
       "epoch": 23.0,
-      "grad_norm": 1.8842490911483765,
+      "grad_norm": 1.7072333097457886,
       "learning_rate": 0.0001098017140329561,
-      "loss": 0.0369,
+      "loss": 0.0621,
       "step": 23
     },
     {
       "epoch": 23.0,
-      "eval_loss": 0.03636971116065979,
-      "eval_runtime": 0.1564,
-      "eval_samples_per_second": 63.924,
-      "eval_steps_per_second": 12.785,
+      "eval_loss": 0.057169489562511444,
+      "eval_runtime": 0.1566,
+      "eval_samples_per_second": 63.85,
+      "eval_steps_per_second": 12.77,
       "step": 23
     },
     {
       "epoch": 24.0,
-      "grad_norm": 0.7897646427154541,
+      "grad_norm": 1.489992380142212,
       "learning_rate": 0.0001,
-      "loss": 0.0332,
+      "loss": 0.0561,
       "step": 24
     },
     {
       "epoch": 24.0,
-      "eval_loss": 0.021218404173851013,
-      "eval_runtime": 0.1562,
-      "eval_samples_per_second": 64.028,
-      "eval_steps_per_second": 12.806,
+      "eval_loss": 0.029277494177222252,
+      "eval_runtime": 0.1557,
+      "eval_samples_per_second": 64.208,
+      "eval_steps_per_second": 12.842,
       "step": 24
     },
     {
       "epoch": 25.0,
-      "grad_norm": 0.7897646427154541,
+      "grad_norm": 1.489992380142212,
       "learning_rate": 9.019828596704394e-05,
-      "loss": 0.0229,
+      "loss": 0.0273,
       "step": 25
     },
     {
       "epoch": 25.0,
-      "eval_loss": 0.021218404173851013,
-      "eval_runtime": 0.1565,
-      "eval_samples_per_second": 63.916,
-      "eval_steps_per_second": 12.783,
+      "eval_loss": 0.029277494177222252,
+      "eval_runtime": 0.155,
+      "eval_samples_per_second": 64.505,
+      "eval_steps_per_second": 12.901,
       "step": 25
     },
     {
       "epoch": 26.0,
-      "grad_norm": 0.553653359413147,
+      "grad_norm": 0.7948157787322998,
       "learning_rate": 8.049096779838719e-05,
-      "loss": 0.0188,
+      "loss": 0.0276,
       "step": 26
     },
     {
       "epoch": 26.0,
-      "eval_loss": 0.010966768488287926,
-      "eval_runtime": 0.1552,
-      "eval_samples_per_second": 64.414,
-      "eval_steps_per_second": 12.883,
+      "eval_loss": 0.01596766524016857,
+      "eval_runtime": 0.1574,
+      "eval_samples_per_second": 63.526,
+      "eval_steps_per_second": 12.705,
       "step": 26
     },
     {
       "epoch": 27.0,
-      "grad_norm": 0.553653359413147,
+      "grad_norm": 0.7948157787322998,
       "learning_rate": 7.097153227455379e-05,
-      "loss": 0.0096,
+      "loss": 0.0141,
       "step": 27
     },
     {
       "epoch": 27.0,
-      "eval_loss": 0.010966768488287926,
-      "eval_runtime": 0.1573,
-      "eval_samples_per_second": 63.588,
-      "eval_steps_per_second": 12.718,
+      "eval_loss": 0.01596766524016857,
+      "eval_runtime": 0.1559,
+      "eval_samples_per_second": 64.154,
+      "eval_steps_per_second": 12.831,
       "step": 27
     },
     {
       "epoch": 28.0,
-      "grad_norm": 0.2734664976596832,
+      "grad_norm": 0.47985532879829407,
       "learning_rate": 6.173165676349103e-05,
-      "loss": 0.0085,
+      "loss": 0.0167,
       "step": 28
     },
     {
       "epoch": 28.0,
-      "eval_loss": 0.010634347796440125,
-      "eval_runtime": 0.1562,
-      "eval_samples_per_second": 64.024,
-      "eval_steps_per_second": 12.805,
+      "eval_loss": 0.0055131325498223305,
+      "eval_runtime": 0.1551,
+      "eval_samples_per_second": 64.494,
+      "eval_steps_per_second": 12.899,
       "step": 28
     },
     {
       "epoch": 29.0,
-      "grad_norm": 0.2734664976596832,
+      "grad_norm": 0.47985532879829407,
       "learning_rate": 5.286032631740023e-05,
-      "loss": 0.0092,
+      "loss": 0.005,
       "step": 29
     },
     {
       "epoch": 29.0,
-      "eval_loss": 0.010634347796440125,
-      "eval_runtime": 0.1573,
-      "eval_samples_per_second": 63.573,
-      "eval_steps_per_second": 12.715,
+      "eval_loss": 0.0055131325498223305,
+      "eval_runtime": 0.1555,
+      "eval_samples_per_second": 64.319,
+      "eval_steps_per_second": 12.864,
       "step": 29
     },
     {
       "epoch": 30.0,
-      "grad_norm": 0.8116740584373474,
+      "grad_norm": 0.2008497714996338,
       "learning_rate": 4.444297669803981e-05,
-      "loss": 0.0086,
+      "loss": 0.0056,
       "step": 30
     },
     {
       "epoch": 30.0,
-      "eval_loss": 0.002800833899527788,
-      "eval_runtime": 0.1563,
-      "eval_samples_per_second": 63.985,
-      "eval_steps_per_second": 12.797,
+      "eval_loss": 0.0027345926500856876,
+      "eval_runtime": 0.1559,
+      "eval_samples_per_second": 64.125,
+      "eval_steps_per_second": 12.825,
       "step": 30
     },
     {
       "epoch": 31.0,
-      "grad_norm": 0.8116740584373474,
+      "grad_norm": 0.2008497714996338,
       "learning_rate": 3.6560671583635467e-05,
-      "loss": 0.0023,
+      "loss": 0.0024,
       "step": 31
     },
     {
       "epoch": 31.0,
-      "eval_loss": 0.002800833899527788,
-      "eval_runtime": 0.1557,
-      "eval_samples_per_second": 64.242,
-      "eval_steps_per_second": 12.848,
+      "eval_loss": 0.0027345926500856876,
+      "eval_runtime": 0.1559,
+      "eval_samples_per_second": 64.15,
+      "eval_steps_per_second": 12.83,
       "step": 31
     },
     {
       "epoch": 32.0,
-      "grad_norm": 0.13856175541877747,
+      "grad_norm": 0.06916889548301697,
       "learning_rate": 2.9289321881345254e-05,
-      "loss": 0.0028,
+      "loss": 0.0026,
       "step": 32
     },
     {
       "epoch": 32.0,
-      "eval_loss": 0.0017185775795951486,
-      "eval_runtime": 0.1572,
-      "eval_samples_per_second": 63.629,
-      "eval_steps_per_second": 12.726,
+      "eval_loss": 0.002255227416753769,
+      "eval_runtime": 0.1552,
+      "eval_samples_per_second": 64.439,
+      "eval_steps_per_second": 12.888,
       "step": 32
     },
     {
       "epoch": 33.0,
-      "grad_norm": 0.13856175541877747,
+      "grad_norm": 0.06916889548301697,
       "learning_rate": 2.26989546637263e-05,
-      "loss": 0.0016,
+      "loss": 0.0022,
       "step": 33
     },
     {
       "epoch": 33.0,
-      "eval_loss": 0.0017185775795951486,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.322,
-      "eval_steps_per_second": 12.864,
+      "eval_loss": 0.002255227416753769,
+      "eval_runtime": 0.1556,
+      "eval_samples_per_second": 64.276,
+      "eval_steps_per_second": 12.855,
       "step": 33
     },
     {
       "epoch": 34.0,
-      "grad_norm": 0.050298936665058136,
+      "grad_norm": 0.0823979377746582,
       "learning_rate": 1.6853038769745467e-05,
-      "loss": 0.0016,
+      "loss": 0.0021,
       "step": 34
     },
     {
       "epoch": 34.0,
-      "eval_loss": 0.0014206025516614318,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.317,
-      "eval_steps_per_second": 12.863,
+      "eval_loss": 0.0017385013634338975,
+      "eval_runtime": 0.1552,
+      "eval_samples_per_second": 64.422,
+      "eval_steps_per_second": 12.884,
       "step": 34
     },
     {
       "epoch": 35.0,
-      "grad_norm": 0.050298936665058136,
+      "grad_norm": 0.0823979377746582,
       "learning_rate": 1.1807873565164506e-05,
-      "loss": 0.0013,
+      "loss": 0.0015,
       "step": 35
     },
     {
       "epoch": 35.0,
-      "eval_loss": 0.0014206025516614318,
-      "eval_runtime": 0.1566,
-      "eval_samples_per_second": 63.862,
-      "eval_steps_per_second": 12.772,
+      "eval_loss": 0.0017385013634338975,
+      "eval_runtime": 0.1555,
+      "eval_samples_per_second": 64.296,
+      "eval_steps_per_second": 12.859,
       "step": 35
     },
     {
       "epoch": 36.0,
-      "grad_norm": 0.04518803581595421,
+      "grad_norm": 0.05911393463611603,
       "learning_rate": 7.612046748871327e-06,
-      "loss": 0.0013,
+      "loss": 0.0016,
       "step": 36
     },
     {
       "epoch": 36.0,
-      "eval_loss": 0.0012987955706194043,
-      "eval_runtime": 0.1581,
-      "eval_samples_per_second": 63.253,
-      "eval_steps_per_second": 12.651,
+      "eval_loss": 0.0014259777963161469,
+      "eval_runtime": 0.1562,
+      "eval_samples_per_second": 64.033,
+      "eval_steps_per_second": 12.807,
       "step": 36
     },
     {
       "epoch": 37.0,
-      "grad_norm": 0.04518803581595421,
+      "grad_norm": 0.05911393463611603,
       "learning_rate": 4.305966426779118e-06,
       "loss": 0.0013,
       "step": 37
     },
     {
       "epoch": 37.0,
-      "eval_loss": 0.0012987955706194043,
-      "eval_runtime": 0.1552,
-      "eval_samples_per_second": 64.423,
-      "eval_steps_per_second": 12.885,
+      "eval_loss": 0.0014259777963161469,
+      "eval_runtime": 0.1553,
+      "eval_samples_per_second": 64.39,
+      "eval_steps_per_second": 12.878,
       "step": 37
     },
     {
       "epoch": 38.0,
-      "grad_norm": 0.04950498044490814,
+      "grad_norm": 0.043380431830883026,
       "learning_rate": 1.921471959676957e-06,
       "loss": 0.0012,
       "step": 38
     },
     {
       "epoch": 38.0,
-      "eval_loss": 0.0012480242876335979,
-      "eval_runtime": 0.1557,
-      "eval_samples_per_second": 64.216,
-      "eval_steps_per_second": 12.843,
+      "eval_loss": 0.001306904829107225,
+      "eval_runtime": 0.1552,
+      "eval_samples_per_second": 64.441,
+      "eval_steps_per_second": 12.888,
       "step": 38
     },
     {
       "epoch": 39.0,
-      "grad_norm": 0.04950498044490814,
+      "grad_norm": 0.043380431830883026,
       "learning_rate": 4.815273327803182e-07,
-      "loss": 0.0012,
+      "loss": 0.0011,
       "step": 39
     },
     {
       "epoch": 39.0,
-      "eval_loss": 0.0012480242876335979,
-      "eval_runtime": 0.1593,
-      "eval_samples_per_second": 62.756,
-      "eval_steps_per_second": 12.551,
+      "eval_loss": 0.001306904829107225,
+      "eval_runtime": 0.155,
+      "eval_samples_per_second": 64.517,
+      "eval_steps_per_second": 12.903,
       "step": 39
     },
     {
       "epoch": 40.0,
-      "grad_norm": 0.05591588839888573,
+      "grad_norm": 0.03869909420609474,
       "learning_rate": 0.0,
       "loss": 0.0011,
       "step": 40
     },
     {
       "epoch": 40.0,
-      "eval_loss": 0.0012466020416468382,
-      "eval_runtime": 0.1587,
-      "eval_samples_per_second": 63.027,
-      "eval_steps_per_second": 12.605,
+      "eval_loss": 0.0013022046769037843,
+      "eval_runtime": 0.1569,
+      "eval_samples_per_second": 63.717,
+      "eval_steps_per_second": 12.743,
       "step": 40
     },
     {
       "epoch": 40.0,
       "step": 40,
       "total_flos": 4849620932886528.0,
-      "train_loss": 0.6959351973579032,
-      "train_runtime": 31.7849,
-      "train_samples_per_second": 12.585,
-      "train_steps_per_second": 1.258
+      "train_loss": 0.7576449837215478,
+      "train_runtime": 31.485,
+      "train_samples_per_second": 12.704,
+      "train_steps_per_second": 1.27
     }
   ],
   "logging_steps": 1,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba9b06b8bcd73c20d1ab29d48efa07a9c52ce282d69ea2221de1b9ccde6ccb65
+oid sha256:d5e1a1784a335e61d9c767f95796afd836f1905ed40a45f46615eadf9c1ef1cc
 size 6968