End of training

all_results.json

@@ -1,19 +1,19 @@
 {
     "epoch": 3.0,
-    "eval_bleu": 0.2510920700175718,
-    "eval_loss": 2.5200693607330322,
-    "eval_rouge1": 0.45339120041307046,
-    "eval_rouge2": 0.21369562389429725,
-    "eval_rougeL": 0.3983722007696566,
-    "eval_runtime": 28.0888,
-    "eval_samples": 884,
-    "eval_samples_per_second": 31.472,
-    "eval_steps_per_second": 3.952,
-    "perplexity": 12.42945875004982,
-    "total_flos": 4151799742464000.0,
-    "train_loss": 3.14277273785115,
-    "train_runtime": 586.188,
-    "train_samples": 3531,
-    "train_samples_per_second": 18.071,
-    "train_steps_per_second": 2.262
+    "eval_bleu": 0.22465329922096197,
+    "eval_loss": 0.758726954460144,
+    "eval_rouge1": 0.5688224315923487,
+    "eval_rouge2": 0.3116501353023259,
+    "eval_rougeL": 0.56490169958113,
+    "eval_runtime": 45.9889,
+    "eval_samples": 14212,
+    "eval_samples_per_second": 309.031,
+    "eval_steps_per_second": 38.64,
+    "perplexity": 2.135555829449579,
+    "total_flos": 1.1141034983424e+16,
+    "train_loss": 0.8646288613632099,
+    "train_runtime": 2092.8708,
+    "train_samples": 56851,
+    "train_samples_per_second": 81.492,
+    "train_steps_per_second": 10.187
 }

egy_training_log.txt

@@ -1394,3 +1394,5 @@ INFO:root:Epoch 2.0: Train Loss = 1.148, Eval Loss = 0.8351342082023621
 INFO:absl:Using default tokenizer.
 INFO:root:Epoch 3.0: Train Loss = 0.7647, Eval Loss = 0.7705017924308777
 INFO:absl:Using default tokenizer.
+INFO:__main__:*** Evaluate ***
+INFO:absl:Using default tokenizer.

eval_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 3.0,
-    "eval_bleu": 0.2510920700175718,
-    "eval_loss": 2.5200693607330322,
-    "eval_rouge1": 0.45339120041307046,
-    "eval_rouge2": 0.21369562389429725,
-    "eval_rougeL": 0.3983722007696566,
-    "eval_runtime": 28.0888,
-    "eval_samples": 884,
-    "eval_samples_per_second": 31.472,
-    "eval_steps_per_second": 3.952,
-    "perplexity": 12.42945875004982
+    "eval_bleu": 0.22465329922096197,
+    "eval_loss": 0.758726954460144,
+    "eval_rouge1": 0.5688224315923487,
+    "eval_rouge2": 0.3116501353023259,
+    "eval_rougeL": 0.56490169958113,
+    "eval_runtime": 45.9889,
+    "eval_samples": 14212,
+    "eval_samples_per_second": 309.031,
+    "eval_steps_per_second": 38.64,
+    "perplexity": 2.135555829449579
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 3.0,
-    "total_flos": 4151799742464000.0,
-    "train_loss": 3.14277273785115,
-    "train_runtime": 586.188,
-    "train_samples": 3531,
-    "train_samples_per_second": 18.071,
-    "train_steps_per_second": 2.262
+    "total_flos": 1.1141034983424e+16,
+    "train_loss": 0.8646288613632099,
+    "train_runtime": 2092.8708,
+    "train_samples": 56851,
+    "train_samples_per_second": 81.492,
+    "train_steps_per_second": 10.187
 }

trainer_state.json

@@ -1,82 +1,82 @@
 {
-  "best_metric": 2.5200693607330322,
-  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_egy/checkpoint-1326",
+  "best_metric": 0.758726954460144,
+  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_egy/checkpoint-21321",
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 1326,
+  "global_step": 21321,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 1.6992123126983643,
-      "learning_rate": 4.4200000000000004e-05,
-      "loss": 4.0723,
-      "step": 442
+      "grad_norm": 2.9758260250091553,
+      "learning_rate": 3.413380721387061e-05,
+      "loss": 1.148,
+      "step": 7107
     },
     {
       "epoch": 1.0,
-      "eval_bleu": 0.22275841970876573,
-      "eval_loss": 2.958087921142578,
-      "eval_rouge1": 0.3912678429308313,
-      "eval_rouge2": 0.16481568948149655,
-      "eval_rougeL": 0.32008248461173605,
-      "eval_runtime": 27.8657,
-      "eval_samples_per_second": 31.724,
-      "eval_steps_per_second": 3.983,
-      "step": 442
+      "eval_bleu": 0.18719204852263469,
+      "eval_loss": 0.8351342082023621,
+      "eval_rouge1": 0.5142833565402041,
+      "eval_rouge2": 0.2505733532313653,
+      "eval_rougeL": 0.5096825616972613,
+      "eval_runtime": 169.5172,
+      "eval_samples_per_second": 83.838,
+      "eval_steps_per_second": 10.483,
+      "step": 7107
     },
     {
       "epoch": 2.0,
-      "grad_norm": 1.676450252532959,
-      "learning_rate": 2.6755447941888623e-05,
-      "loss": 2.8055,
-      "step": 884
+      "grad_norm": 3.232348680496216,
+      "learning_rate": 1.7066903606935305e-05,
+      "loss": 0.7647,
+      "step": 14214
     },
     {
       "epoch": 2.0,
-      "eval_bleu": 0.24154624358915514,
-      "eval_loss": 2.606330394744873,
-      "eval_rouge1": 0.43470628711649895,
-      "eval_rouge2": 0.19710102219495218,
-      "eval_rougeL": 0.3776869994743499,
-      "eval_runtime": 28.1779,
-      "eval_samples_per_second": 31.372,
-      "eval_steps_per_second": 3.939,
-      "step": 884
+      "eval_bleu": 0.2145574961692684,
+      "eval_loss": 0.7705017924308777,
+      "eval_rouge1": 0.5551766890409657,
+      "eval_rouge2": 0.29664921512752485,
+      "eval_rougeL": 0.5514002756721588,
+      "eval_runtime": 108.2094,
+      "eval_samples_per_second": 131.338,
+      "eval_steps_per_second": 16.422,
+      "step": 14214
     },
     {
       "epoch": 3.0,
-      "grad_norm": 1.5551155805587769,
+      "grad_norm": 2.0039141178131104,
       "learning_rate": 0.0,
-      "loss": 2.5505,
-      "step": 1326
+      "loss": 0.6812,
+      "step": 21321
     },
     {
       "epoch": 3.0,
-      "eval_bleu": 0.2510920700175718,
-      "eval_loss": 2.5200693607330322,
-      "eval_rouge1": 0.45339120041307046,
-      "eval_rouge2": 0.21369562389429725,
-      "eval_rougeL": 0.3983722007696566,
-      "eval_runtime": 28.2169,
-      "eval_samples_per_second": 31.329,
-      "eval_steps_per_second": 3.934,
-      "step": 1326
+      "eval_bleu": 0.22465329922096197,
+      "eval_loss": 0.758726954460144,
+      "eval_rouge1": 0.5688224315923487,
+      "eval_rouge2": 0.3116501353023259,
+      "eval_rougeL": 0.56490169958113,
+      "eval_runtime": 292.0707,
+      "eval_samples_per_second": 48.659,
+      "eval_steps_per_second": 6.084,
+      "step": 21321
     },
     {
       "epoch": 3.0,
-      "step": 1326,
-      "total_flos": 4151799742464000.0,
-      "train_loss": 3.14277273785115,
-      "train_runtime": 586.188,
-      "train_samples_per_second": 18.071,
-      "train_steps_per_second": 2.262
+      "step": 21321,
+      "total_flos": 1.1141034983424e+16,
+      "train_loss": 0.8646288613632099,
+      "train_runtime": 2092.8708,
+      "train_samples_per_second": 81.492,
+      "train_steps_per_second": 10.187
     }
   ],
   "logging_steps": 500,
-  "max_steps": 1326,
+  "max_steps": 21321,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
@@ -101,7 +101,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4151799742464000.0,
+  "total_flos": 1.1141034983424e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null