Model save

README.md

@@ -4,7 +4,6 @@ library_name: transformers
 model_name: llama_tuned
 tags:
 - generated_from_trainer
-- alignment-handbook
 - trl
 - sft
 licence: license
@@ -28,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/vedaantj/ft-llm/runs/00gc56v6)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/vedaantj/ft-llm/runs/0o7eyh1v)
 
 This model was trained with SFT.
 
@@ -36,8 +35,8 @@ This model was trained with SFT.
 
 - TRL: 0.12.2
 - Transformers: 4.46.3
-- Pytorch: 2.4.1
-- Datasets: 3.1.0
+- Pytorch: 2.4.1+cu124
+- Datasets: 3.2.0
 - Tokenizers: 0.20.3
 
 ## Citations

all_results.json

@@ -1,14 +1,9 @@
 {
-    "epoch": 40.0,
-    "eval_loss": 0.0030517119448632,
-    "eval_runtime": 0.1724,
-    "eval_samples": 10,
-    "eval_samples_per_second": 58.01,
-    "eval_steps_per_second": 11.602,
-    "total_flos": 4882092898910208.0,
-    "train_loss": 0.5672883474326227,
-    "train_runtime": 31.3668,
-    "train_samples": 10,
-    "train_samples_per_second": 12.752,
-    "train_steps_per_second": 1.275
+    "epoch": 39.89333333333333,
+    "total_flos": 1.743333613371392e+18,
+    "train_loss": 0.10288844243324496,
+    "train_runtime": 9416.1259,
+    "train_samples": 4498,
+    "train_samples_per_second": 19.108,
+    "train_steps_per_second": 0.794
 }

config.json

@@ -22,7 +22,7 @@
   "num_attention_heads": 24,
   "num_hidden_layers": 28,
   "num_key_value_heads": 8,
-  "pad_token_id": 128009,
+  "pad_token_id": 128002,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
@@ -36,6 +36,6 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.46.3",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 128256
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:633bf1ca74fc04074527717859d80919eccf368ed25c2972229a37d4ee376b11
+oid sha256:829c5d3be53b79d41cde0fc8920a3f9575feb4515076ec33263735a3465cafe5
 size 4965799096

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67bb5340c0a30c7b09d3e885637cc10a84a99e891ddc1b37a3b2ca062d7e5c23
+oid sha256:ea7eebbe9bb65775e6fdad67305d7f5cf2f8108944b253cd942cd21cf3ae6640
 size 2247734992

special_tokens_map.json

@@ -13,5 +13,11 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|eot_id|>"
+  "pad_token": {
+    "content": "<|reserved_special_token_0|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -2058,6 +2058,6 @@
     "attention_mask"
   ],
   "model_max_length": 2048,
-  "pad_token": "<|eot_id|>",
+  "pad_token": "<|reserved_special_token_0|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 40.0,
-    "total_flos": 4882092898910208.0,
-    "train_loss": 0.5672883474326227,
-    "train_runtime": 31.3668,
-    "train_samples": 10,
-    "train_samples_per_second": 12.752,
-    "train_steps_per_second": 1.275
+    "epoch": 39.89333333333333,
+    "total_flos": 1.743333613371392e+18,
+    "train_loss": 0.10288844243324496,
+    "train_runtime": 9416.1259,
+    "train_samples": 4498,
+    "train_samples_per_second": 19.108,
+    "train_steps_per_second": 0.794
 }

trainer_state.json

@@ -1,624 +1,422 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 40.0,
+  "epoch": 39.89333333333333,
   "eval_steps": 500,
-  "global_step": 40,
+  "global_step": 7480,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "learning_rate": 2.5e-05,
-      "loss": 1.1277,
+      "epoch": 0.005333333333333333,
+      "grad_norm": 3.9845798015594482,
+      "learning_rate": 1.3368983957219251e-07,
+      "loss": 1.2155,
       "step": 1
     },
     {
-      "epoch": 1.0,
-      "eval_loss": 1.131566047668457,
-      "eval_runtime": 0.1598,
-      "eval_samples_per_second": 62.597,
-      "eval_steps_per_second": 12.519,
-      "step": 1
+      "epoch": 0.9973333333333333,
+      "eval_loss": 0.6746753454208374,
+      "eval_runtime": 12.9251,
+      "eval_samples_per_second": 86.963,
+      "eval_steps_per_second": 14.545,
+      "step": 187
     },
     {
       "epoch": 2.0,
-      "grad_norm": 9.749205589294434,
-      "learning_rate": 5e-05,
-      "loss": 1.1065,
-      "step": 2
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 0.7347294092178345,
-      "eval_runtime": 0.1573,
-      "eval_samples_per_second": 63.584,
-      "eval_steps_per_second": 12.717,
-      "step": 2
-    },
-    {
-      "epoch": 3.0,
-      "grad_norm": 9.749205589294434,
-      "learning_rate": 7.500000000000001e-05,
-      "loss": 0.7329,
-      "step": 3
+      "eval_loss": 0.7129796147346497,
+      "eval_runtime": 12.5123,
+      "eval_samples_per_second": 89.831,
+      "eval_steps_per_second": 15.025,
+      "step": 375
     },
     {
-      "epoch": 3.0,
-      "eval_loss": 0.7347294092178345,
-      "eval_runtime": 0.1562,
-      "eval_samples_per_second": 64.01,
-      "eval_steps_per_second": 12.802,
-      "step": 3
+      "epoch": 2.997333333333333,
+      "eval_loss": 0.8005833029747009,
+      "eval_runtime": 12.5146,
+      "eval_samples_per_second": 89.815,
+      "eval_steps_per_second": 15.022,
+      "step": 562
     },
     {
-      "epoch": 4.0,
-      "grad_norm": 6.555100440979004,
-      "learning_rate": 0.0001,
-      "loss": 0.7406,
-      "step": 4
+      "epoch": 3.7333333333333334,
+      "grad_norm": 0.7766222357749939,
+      "learning_rate": 9.358288770053476e-05,
+      "loss": 0.582,
+      "step": 700
     },
     {
       "epoch": 4.0,
-      "eval_loss": 1.1057980060577393,
-      "eval_runtime": 0.1567,
-      "eval_samples_per_second": 63.83,
-      "eval_steps_per_second": 12.766,
-      "step": 4
+      "eval_loss": 0.8906850814819336,
+      "eval_runtime": 12.5313,
+      "eval_samples_per_second": 89.695,
+      "eval_steps_per_second": 15.002,
+      "step": 750
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 6.555100440979004,
-      "learning_rate": 0.000125,
-      "loss": 1.0826,
-      "step": 5
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 1.1057980060577393,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.313,
-      "eval_steps_per_second": 12.863,
-      "step": 5
+      "epoch": 4.997333333333334,
+      "eval_loss": 0.9965974688529968,
+      "eval_runtime": 12.4874,
+      "eval_samples_per_second": 90.011,
+      "eval_steps_per_second": 15.055,
+      "step": 937
     },
     {
       "epoch": 6.0,
-      "grad_norm": 18.290578842163086,
-      "learning_rate": 0.00015000000000000001,
-      "loss": 1.0478,
-      "step": 6
+      "eval_loss": 1.1142816543579102,
+      "eval_runtime": 12.5004,
+      "eval_samples_per_second": 89.917,
+      "eval_steps_per_second": 15.039,
+      "step": 1125
     },
     {
-      "epoch": 6.0,
-      "eval_loss": 1.3313286304473877,
-      "eval_runtime": 0.1563,
-      "eval_samples_per_second": 63.987,
-      "eval_steps_per_second": 12.797,
-      "step": 6
+      "epoch": 6.997333333333334,
+      "eval_loss": 1.200727939605713,
+      "eval_runtime": 12.5034,
+      "eval_samples_per_second": 89.896,
+      "eval_steps_per_second": 15.036,
+      "step": 1312
     },
     {
-      "epoch": 7.0,
-      "grad_norm": 18.290578842163086,
-      "learning_rate": 0.000175,
-      "loss": 1.3493,
-      "step": 7
-    },
-    {
-      "epoch": 7.0,
-      "eval_loss": 1.3313286304473877,
-      "eval_runtime": 0.1554,
-      "eval_samples_per_second": 64.364,
-      "eval_steps_per_second": 12.873,
-      "step": 7
-    },
-    {
-      "epoch": 8.0,
-      "grad_norm": 51.25260925292969,
-      "learning_rate": 0.0002,
-      "loss": 1.4307,
-      "step": 8
+      "epoch": 7.466666666666667,
+      "grad_norm": 0.6246238946914673,
+      "learning_rate": 0.0001871657754010695,
+      "loss": 0.2905,
+      "step": 1400
     },
     {
       "epoch": 8.0,
-      "eval_loss": 0.7719938158988953,
-      "eval_runtime": 0.1571,
-      "eval_samples_per_second": 63.674,
-      "eval_steps_per_second": 12.735,
-      "step": 8
+      "eval_loss": 1.2675864696502686,
+      "eval_runtime": 12.484,
+      "eval_samples_per_second": 90.035,
+      "eval_steps_per_second": 15.059,
+      "step": 1500
     },
     {
-      "epoch": 9.0,
-      "grad_norm": 51.25260925292969,
-      "learning_rate": 0.0001995184726672197,
-      "loss": 0.7569,
-      "step": 9
-    },
-    {
-      "epoch": 9.0,
-      "eval_loss": 0.7719938158988953,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.326,
-      "eval_steps_per_second": 12.865,
-      "step": 9
+      "epoch": 8.997333333333334,
+      "eval_loss": 1.3635159730911255,
+      "eval_runtime": 12.4913,
+      "eval_samples_per_second": 89.983,
+      "eval_steps_per_second": 15.05,
+      "step": 1687
     },
     {
       "epoch": 10.0,
-      "grad_norm": 7.65546178817749,
-      "learning_rate": 0.00019807852804032305,
-      "loss": 0.785,
-      "step": 10
+      "eval_loss": 1.4521616697311401,
+      "eval_runtime": 12.5047,
+      "eval_samples_per_second": 89.886,
+      "eval_steps_per_second": 15.034,
+      "step": 1875
     },
     {
-      "epoch": 10.0,
-      "eval_loss": 3.317613124847412,
-      "eval_runtime": 0.1563,
-      "eval_samples_per_second": 63.973,
-      "eval_steps_per_second": 12.795,
-      "step": 10
+      "epoch": 10.997333333333334,
+      "eval_loss": 1.4860386848449707,
+      "eval_runtime": 12.4767,
+      "eval_samples_per_second": 90.088,
+      "eval_steps_per_second": 15.068,
+      "step": 2062
     },
     {
-      "epoch": 11.0,
-      "grad_norm": 7.65546178817749,
-      "learning_rate": 0.0001956940335732209,
-      "loss": 3.3605,
-      "step": 11
-    },
-    {
-      "epoch": 11.0,
-      "eval_loss": 3.317613124847412,
-      "eval_runtime": 0.1553,
-      "eval_samples_per_second": 64.372,
-      "eval_steps_per_second": 12.874,
-      "step": 11
+      "epoch": 11.2,
+      "grad_norm": 0.3173273503780365,
+      "learning_rate": 0.00019501438995110992,
+      "loss": 0.1313,
+      "step": 2100
     },
     {
       "epoch": 12.0,
-      "grad_norm": 132.75279235839844,
-      "learning_rate": 0.0001923879532511287,
-      "loss": 3.4664,
-      "step": 12
+      "eval_loss": 1.5470738410949707,
+      "eval_runtime": 12.4572,
+      "eval_samples_per_second": 90.229,
+      "eval_steps_per_second": 15.092,
+      "step": 2250
     },
     {
-      "epoch": 12.0,
-      "eval_loss": 1.0249794721603394,
-      "eval_runtime": 0.1586,
-      "eval_samples_per_second": 63.039,
-      "eval_steps_per_second": 12.608,
-      "step": 12
-    },
-    {
-      "epoch": 13.0,
-      "grad_norm": 132.75279235839844,
-      "learning_rate": 0.0001881921264348355,
-      "loss": 1.0922,
-      "step": 13
-    },
-    {
-      "epoch": 13.0,
-      "eval_loss": 1.0249794721603394,
-      "eval_runtime": 0.1558,
-      "eval_samples_per_second": 64.204,
-      "eval_steps_per_second": 12.841,
-      "step": 13
-    },
-    {
-      "epoch": 14.0,
-      "grad_norm": 10.862001419067383,
-      "learning_rate": 0.00018314696123025454,
-      "loss": 1.0792,
-      "step": 14
+      "epoch": 12.997333333333334,
+      "eval_loss": 1.5745112895965576,
+      "eval_runtime": 12.4452,
+      "eval_samples_per_second": 90.316,
+      "eval_steps_per_second": 15.106,
+      "step": 2437
     },
     {
       "epoch": 14.0,
-      "eval_loss": 0.7917268872261047,
-      "eval_runtime": 0.1564,
-      "eval_samples_per_second": 63.95,
-      "eval_steps_per_second": 12.79,
-      "step": 14
+      "eval_loss": 1.6154626607894897,
+      "eval_runtime": 12.4425,
+      "eval_samples_per_second": 90.335,
+      "eval_steps_per_second": 15.109,
+      "step": 2625
     },
     {
-      "epoch": 15.0,
-      "grad_norm": 10.862001419067383,
-      "learning_rate": 0.0001773010453362737,
-      "loss": 0.8238,
-      "step": 15
+      "epoch": 14.933333333333334,
+      "grad_norm": 0.269999235868454,
+      "learning_rate": 0.000177467306667847,
+      "loss": 0.0448,
+      "step": 2800
     },
     {
-      "epoch": 15.0,
-      "eval_loss": 0.7917268872261047,
-      "eval_runtime": 0.1557,
-      "eval_samples_per_second": 64.219,
-      "eval_steps_per_second": 12.844,
-      "step": 15
+      "epoch": 14.997333333333334,
+      "eval_loss": 1.6521568298339844,
+      "eval_runtime": 12.453,
+      "eval_samples_per_second": 90.26,
+      "eval_steps_per_second": 15.097,
+      "step": 2812
     },
     {
       "epoch": 16.0,
-      "grad_norm": 20.844392776489258,
-      "learning_rate": 0.00017071067811865476,
-      "loss": 0.8054,
-      "step": 16
+      "eval_loss": 1.693098783493042,
+      "eval_runtime": 12.4351,
+      "eval_samples_per_second": 90.39,
+      "eval_steps_per_second": 15.119,
+      "step": 3000
     },
     {
-      "epoch": 16.0,
-      "eval_loss": 0.3446378707885742,
-      "eval_runtime": 0.1553,
-      "eval_samples_per_second": 64.409,
-      "eval_steps_per_second": 12.882,
-      "step": 16
-    },
-    {
-      "epoch": 17.0,
-      "grad_norm": 20.844392776489258,
-      "learning_rate": 0.00016343932841636456,
-      "loss": 0.3504,
-      "step": 17
-    },
-    {
-      "epoch": 17.0,
-      "eval_loss": 0.3446378707885742,
-      "eval_runtime": 0.157,
-      "eval_samples_per_second": 63.679,
-      "eval_steps_per_second": 12.736,
-      "step": 17
+      "epoch": 16.997333333333334,
+      "eval_loss": 1.7404099702835083,
+      "eval_runtime": 12.4667,
+      "eval_samples_per_second": 90.16,
+      "eval_steps_per_second": 15.08,
+      "step": 3187
     },
     {
       "epoch": 18.0,
-      "grad_norm": 3.432838201522827,
-      "learning_rate": 0.00015555702330196023,
-      "loss": 0.3424,
-      "step": 18
-    },
-    {
-      "epoch": 18.0,
-      "eval_loss": 0.27769702672958374,
-      "eval_runtime": 0.1557,
-      "eval_samples_per_second": 64.21,
-      "eval_steps_per_second": 12.842,
-      "step": 18
-    },
-    {
-      "epoch": 19.0,
-      "grad_norm": 3.432838201522827,
-      "learning_rate": 0.0001471396736825998,
-      "loss": 0.2479,
-      "step": 19
+      "eval_loss": 1.7414445877075195,
+      "eval_runtime": 12.4548,
+      "eval_samples_per_second": 90.246,
+      "eval_steps_per_second": 15.095,
+      "step": 3375
     },
     {
-      "epoch": 19.0,
-      "eval_loss": 0.27769702672958374,
-      "eval_runtime": 0.1567,
-      "eval_samples_per_second": 63.823,
-      "eval_steps_per_second": 12.765,
-      "step": 19
+      "epoch": 18.666666666666668,
+      "grad_norm": 0.16562360525131226,
+      "learning_rate": 0.0001495750499965753,
+      "loss": 0.0268,
+      "step": 3500
     },
     {
-      "epoch": 20.0,
-      "grad_norm": 2.582324981689453,
-      "learning_rate": 0.000138268343236509,
-      "loss": 0.244,
-      "step": 20
+      "epoch": 18.997333333333334,
+      "eval_loss": 1.7563754320144653,
+      "eval_runtime": 12.9626,
+      "eval_samples_per_second": 86.711,
+      "eval_steps_per_second": 14.503,
+      "step": 3562
     },
     {
       "epoch": 20.0,
-      "eval_loss": 0.1163192018866539,
-      "eval_runtime": 0.1566,
-      "eval_samples_per_second": 63.838,
-      "eval_steps_per_second": 12.768,
-      "step": 20
-    },
-    {
-      "epoch": 21.0,
-      "grad_norm": 2.582324981689453,
-      "learning_rate": 0.00012902846772544624,
-      "loss": 0.1097,
-      "step": 21
-    },
-    {
-      "epoch": 21.0,
-      "eval_loss": 0.1163192018866539,
-      "eval_runtime": 0.1555,
-      "eval_samples_per_second": 64.297,
-      "eval_steps_per_second": 12.859,
-      "step": 21
+      "eval_loss": 1.768137812614441,
+      "eval_runtime": 12.4413,
+      "eval_samples_per_second": 90.344,
+      "eval_steps_per_second": 15.111,
+      "step": 3750
     },
     {
-      "epoch": 22.0,
-      "grad_norm": 1.2364883422851562,
-      "learning_rate": 0.00011950903220161285,
-      "loss": 0.1126,
-      "step": 22
+      "epoch": 20.997333333333334,
+      "eval_loss": 1.7924139499664307,
+      "eval_runtime": 12.441,
+      "eval_samples_per_second": 90.346,
+      "eval_steps_per_second": 15.111,
+      "step": 3937
     },
     {
       "epoch": 22.0,
-      "eval_loss": 0.0716390460729599,
-      "eval_runtime": 0.1561,
-      "eval_samples_per_second": 64.064,
-      "eval_steps_per_second": 12.813,
-      "step": 22
+      "eval_loss": 1.832397222518921,
+      "eval_runtime": 12.4418,
+      "eval_samples_per_second": 90.34,
+      "eval_steps_per_second": 15.11,
+      "step": 4125
     },
     {
-      "epoch": 23.0,
-      "grad_norm": 1.2364883422851562,
-      "learning_rate": 0.0001098017140329561,
-      "loss": 0.0797,
-      "step": 23
+      "epoch": 22.4,
+      "grad_norm": 0.10293902456760406,
+      "learning_rate": 0.0001150624200350232,
+      "loss": 0.0157,
+      "step": 4200
     },
     {
-      "epoch": 23.0,
-      "eval_loss": 0.0716390460729599,
-      "eval_runtime": 0.1579,
-      "eval_samples_per_second": 63.343,
-      "eval_steps_per_second": 12.669,
-      "step": 23
-    },
-    {
-      "epoch": 24.0,
-      "grad_norm": 1.081320881843567,
-      "learning_rate": 0.0001,
-      "loss": 0.0669,
-      "step": 24
+      "epoch": 22.997333333333334,
+      "eval_loss": 1.8358964920043945,
+      "eval_runtime": 12.424,
+      "eval_samples_per_second": 90.47,
+      "eval_steps_per_second": 15.132,
+      "step": 4312
     },
     {
       "epoch": 24.0,
-      "eval_loss": 0.05489348620176315,
-      "eval_runtime": 0.1558,
-      "eval_samples_per_second": 64.175,
-      "eval_steps_per_second": 12.835,
-      "step": 24
+      "eval_loss": 1.8424901962280273,
+      "eval_runtime": 12.4549,
+      "eval_samples_per_second": 90.246,
+      "eval_steps_per_second": 15.094,
+      "step": 4500
     },
     {
-      "epoch": 25.0,
-      "grad_norm": 1.081320881843567,
-      "learning_rate": 9.019828596704394e-05,
-      "loss": 0.0561,
-      "step": 25
-    },
-    {
-      "epoch": 25.0,
-      "eval_loss": 0.05489348620176315,
-      "eval_runtime": 0.1575,
-      "eval_samples_per_second": 63.477,
-      "eval_steps_per_second": 12.695,
-      "step": 25
-    },
-    {
-      "epoch": 26.0,
-      "grad_norm": 1.572256326675415,
-      "learning_rate": 8.049096779838719e-05,
-      "loss": 0.0569,
-      "step": 26
+      "epoch": 24.997333333333334,
+      "eval_loss": 1.8898138999938965,
+      "eval_runtime": 12.4368,
+      "eval_samples_per_second": 90.377,
+      "eval_steps_per_second": 15.116,
+      "step": 4687
     },
     {
       "epoch": 26.0,
-      "eval_loss": 0.03670552000403404,
-      "eval_runtime": 0.1561,
-      "eval_samples_per_second": 64.056,
-      "eval_steps_per_second": 12.811,
-      "step": 26
+      "eval_loss": 1.9125876426696777,
+      "eval_runtime": 12.4412,
+      "eval_samples_per_second": 90.345,
+      "eval_steps_per_second": 15.111,
+      "step": 4875
     },
     {
-      "epoch": 27.0,
-      "grad_norm": 1.572256326675415,
-      "learning_rate": 7.097153227455379e-05,
-      "loss": 0.0357,
-      "step": 27
+      "epoch": 26.133333333333333,
+      "grad_norm": 0.07291631400585175,
+      "learning_rate": 7.853831770392755e-05,
+      "loss": 0.0061,
+      "step": 4900
     },
     {
-      "epoch": 27.0,
-      "eval_loss": 0.03670552000403404,
-      "eval_runtime": 0.1559,
-      "eval_samples_per_second": 64.161,
-      "eval_steps_per_second": 12.832,
-      "step": 27
+      "epoch": 26.997333333333334,
+      "eval_loss": 1.9254367351531982,
+      "eval_runtime": 12.4413,
+      "eval_samples_per_second": 90.345,
+      "eval_steps_per_second": 15.111,
+      "step": 5062
     },
     {
       "epoch": 28.0,
-      "grad_norm": 0.9304390549659729,
-      "learning_rate": 6.173165676349103e-05,
-      "loss": 0.0331,
-      "step": 28
+      "eval_loss": 1.975632667541504,
+      "eval_runtime": 12.4478,
+      "eval_samples_per_second": 90.297,
+      "eval_steps_per_second": 15.103,
+      "step": 5250
     },
     {
-      "epoch": 28.0,
-      "eval_loss": 0.05289148539304733,
-      "eval_runtime": 0.1561,
-      "eval_samples_per_second": 64.054,
-      "eval_steps_per_second": 12.811,
-      "step": 28
-    },
-    {
-      "epoch": 29.0,
-      "grad_norm": 0.9304390549659729,
-      "learning_rate": 5.286032631740023e-05,
-      "loss": 0.0454,
-      "step": 29
+      "epoch": 28.997333333333334,
+      "eval_loss": 1.9888043403625488,
+      "eval_runtime": 12.4518,
+      "eval_samples_per_second": 90.268,
+      "eval_steps_per_second": 15.098,
+      "step": 5437
     },
     {
-      "epoch": 29.0,
-      "eval_loss": 0.05289148539304733,
-      "eval_runtime": 0.1561,
-      "eval_samples_per_second": 64.068,
-      "eval_steps_per_second": 12.814,
-      "step": 29
+      "epoch": 29.866666666666667,
+      "grad_norm": 0.000998605857603252,
+      "learning_rate": 4.4880260858357746e-05,
+      "loss": 0.0011,
+      "step": 5600
     },
     {
       "epoch": 30.0,
-      "grad_norm": 3.730225086212158,
-      "learning_rate": 4.444297669803981e-05,
-      "loss": 0.0562,
-      "step": 30
+      "eval_loss": 2.0155458450317383,
+      "eval_runtime": 12.4346,
+      "eval_samples_per_second": 90.393,
+      "eval_steps_per_second": 15.119,
+      "step": 5625
     },
     {
-      "epoch": 30.0,
-      "eval_loss": 0.02020808681845665,
-      "eval_runtime": 0.1578,
-      "eval_samples_per_second": 63.372,
-      "eval_steps_per_second": 12.674,
-      "step": 30
-    },
-    {
-      "epoch": 31.0,
-      "grad_norm": 3.730225086212158,
-      "learning_rate": 3.6560671583635467e-05,
-      "loss": 0.0184,
-      "step": 31
-    },
-    {
-      "epoch": 31.0,
-      "eval_loss": 0.02020808681845665,
-      "eval_runtime": 0.156,
-      "eval_samples_per_second": 64.11,
-      "eval_steps_per_second": 12.822,
-      "step": 31
+      "epoch": 30.997333333333334,
+      "eval_loss": 2.048884153366089,
+      "eval_runtime": 12.435,
+      "eval_samples_per_second": 90.39,
+      "eval_steps_per_second": 15.119,
+      "step": 5812
     },
     {
       "epoch": 32.0,
-      "grad_norm": 0.8519256711006165,
-      "learning_rate": 2.9289321881345254e-05,
-      "loss": 0.0186,
-      "step": 32
-    },
-    {
-      "epoch": 32.0,
-      "eval_loss": 0.006217154674232006,
-      "eval_runtime": 0.1564,
-      "eval_samples_per_second": 63.957,
-      "eval_steps_per_second": 12.791,
-      "step": 32
-    },
-    {
-      "epoch": 33.0,
-      "grad_norm": 0.8519256711006165,
-      "learning_rate": 2.26989546637263e-05,
-      "loss": 0.0056,
-      "step": 33
+      "eval_loss": 2.070535898208618,
+      "eval_runtime": 12.426,
+      "eval_samples_per_second": 90.455,
+      "eval_steps_per_second": 15.13,
+      "step": 6000
     },
     {
-      "epoch": 33.0,
-      "eval_loss": 0.006217154674232006,
-      "eval_runtime": 0.1582,
-      "eval_samples_per_second": 63.206,
-      "eval_steps_per_second": 12.641,
-      "step": 33
+      "epoch": 32.99733333333333,
+      "eval_loss": 2.087083339691162,
+      "eval_runtime": 12.428,
+      "eval_samples_per_second": 90.441,
+      "eval_steps_per_second": 15.127,
+      "step": 6187
     },
     {
-      "epoch": 34.0,
-      "grad_norm": 0.17823612689971924,
-      "learning_rate": 1.6853038769745467e-05,
-      "loss": 0.0055,
-      "step": 34
+      "epoch": 33.6,
+      "grad_norm": 0.0002252462727483362,
+      "learning_rate": 1.8583028637387568e-05,
+      "loss": 0.0001,
+      "step": 6300
     },
     {
       "epoch": 34.0,
-      "eval_loss": 0.004052485339343548,
-      "eval_runtime": 0.1567,
-      "eval_samples_per_second": 63.821,
-      "eval_steps_per_second": 12.764,
-      "step": 34
-    },
-    {
-      "epoch": 35.0,
-      "grad_norm": 0.17823612689971924,
-      "learning_rate": 1.1807873565164506e-05,
-      "loss": 0.0038,
-      "step": 35
+      "eval_loss": 2.099353551864624,
+      "eval_runtime": 12.4444,
+      "eval_samples_per_second": 90.322,
+      "eval_steps_per_second": 15.107,
+      "step": 6375
     },
     {
-      "epoch": 35.0,
-      "eval_loss": 0.004052485339343548,
-      "eval_runtime": 0.1558,
-      "eval_samples_per_second": 64.205,
-      "eval_steps_per_second": 12.841,
-      "step": 35
+      "epoch": 34.99733333333333,
+      "eval_loss": 2.1083507537841797,
+      "eval_runtime": 12.431,
+      "eval_samples_per_second": 90.419,
+      "eval_steps_per_second": 15.123,
+      "step": 6562
     },
     {
       "epoch": 36.0,
-      "grad_norm": 0.14803585410118103,
-      "learning_rate": 7.612046748871327e-06,
-      "loss": 0.0037,
-      "step": 36
+      "eval_loss": 2.114396333694458,
+      "eval_runtime": 12.8335,
+      "eval_samples_per_second": 87.583,
+      "eval_steps_per_second": 14.649,
+      "step": 6750
     },
     {
-      "epoch": 36.0,
-      "eval_loss": 0.0032847567927092314,
-      "eval_runtime": 0.1562,
-      "eval_samples_per_second": 64.036,
-      "eval_steps_per_second": 12.807,
-      "step": 36
+      "epoch": 36.99733333333333,
+      "eval_loss": 2.1184046268463135,
+      "eval_runtime": 12.4403,
+      "eval_samples_per_second": 90.352,
+      "eval_steps_per_second": 15.112,
+      "step": 6937
     },
     {
-      "epoch": 37.0,
-      "grad_norm": 0.14803585410118103,
-      "learning_rate": 4.305966426779118e-06,
-      "loss": 0.0031,
-      "step": 37
-    },
-    {
-      "epoch": 37.0,
-      "eval_loss": 0.0032847567927092314,
-      "eval_runtime": 0.1553,
-      "eval_samples_per_second": 64.378,
-      "eval_steps_per_second": 12.876,
-      "step": 37
-    },
-    {
-      "epoch": 38.0,
-      "grad_norm": 0.08926349878311157,
-      "learning_rate": 1.921471959676957e-06,
-      "loss": 0.0028,
-      "step": 38
+      "epoch": 37.333333333333336,
+      "grad_norm": 0.00023204727040138096,
+      "learning_rate": 3.1584176769466346e-06,
+      "loss": 0.0,
+      "step": 7000
     },
     {
       "epoch": 38.0,
-      "eval_loss": 0.0030998815782368183,
-      "eval_runtime": 0.1577,
-      "eval_samples_per_second": 63.417,
-      "eval_steps_per_second": 12.683,
-      "step": 38
-    },
-    {
-      "epoch": 39.0,
-      "grad_norm": 0.08926349878311157,
-      "learning_rate": 4.815273327803182e-07,
-      "loss": 0.0028,
-      "step": 39
-    },
-    {
-      "epoch": 39.0,
-      "eval_loss": 0.0030998815782368183,
-      "eval_runtime": 0.1568,
-      "eval_samples_per_second": 63.793,
-      "eval_steps_per_second": 12.759,
-      "step": 39
-    },
-    {
-      "epoch": 40.0,
-      "grad_norm": 0.08725006133317947,
-      "learning_rate": 0.0,
-      "loss": 0.0028,
-      "step": 40
-    },
-    {
-      "epoch": 40.0,
-      "eval_loss": 0.0030517119448632,
-      "eval_runtime": 0.1564,
-      "eval_samples_per_second": 63.934,
-      "eval_steps_per_second": 12.787,
-      "step": 40
-    },
-    {
-      "epoch": 40.0,
-      "step": 40,
-      "total_flos": 4882092898910208.0,
-      "train_loss": 0.5672883474326227,
-      "train_runtime": 31.3668,
-      "train_samples_per_second": 12.752,
-      "train_steps_per_second": 1.275
+      "eval_loss": 2.12043833732605,
+      "eval_runtime": 12.6027,
+      "eval_samples_per_second": 89.187,
+      "eval_steps_per_second": 14.917,
+      "step": 7125
+    },
+    {
+      "epoch": 38.99733333333333,
+      "eval_loss": 2.121260643005371,
+      "eval_runtime": 12.4106,
+      "eval_samples_per_second": 90.568,
+      "eval_steps_per_second": 15.148,
+      "step": 7312
+    },
+    {
+      "epoch": 39.89333333333333,
+      "eval_loss": 2.1213064193725586,
+      "eval_runtime": 12.42,
+      "eval_samples_per_second": 90.499,
+      "eval_steps_per_second": 15.137,
+      "step": 7480
+    },
+    {
+      "epoch": 39.89333333333333,
+      "step": 7480,
+      "total_flos": 1.743333613371392e+18,
+      "train_loss": 0.10288844243324496,
+      "train_runtime": 9416.1259,
+      "train_samples_per_second": 19.108,
+      "train_steps_per_second": 0.794
     }
   ],
-  "logging_steps": 1,
-  "max_steps": 40,
+  "logging_steps": 700,
+  "max_steps": 7480,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 40,
   "save_steps": 500,
@@ -634,7 +432,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4882092898910208.0,
+  "total_flos": 1.743333613371392e+18,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:195513e53529c66c5cbfa2e89f4dcaa806198d6a1a878ca5c626aa8f2e024353
+oid sha256:58cdc9822e98c96c80fdf0c0a44b460d239c799a0f3d92997b46d43348fce84e
 size 6968