Training in progress, step 10000

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 0.354862233609357,
-    "eval_bleu": 26.3346,
-    "eval_gen_len": 26.6907,
-    "eval_loss": 1.6253596544265747,
-    "eval_runtime": 221.231,
+    "epoch": 2.7777777777777777,
+    "eval_bleu": 23.6596,
+    "eval_gen_len": 27.526,
+    "eval_loss": 1.7469114065170288,
+    "eval_runtime": 235.9606,
     "eval_samples": 3000,
-    "eval_samples_per_second": 13.56,
-    "eval_steps_per_second": 1.695,
-    "total_flos": 3.589813132276531e+16,
-    "train_loss": 2.132816611328125,
-    "train_runtime": 15981.9131,
-    "train_samples": 4508785,
-    "train_samples_per_second": 100.113,
-    "train_steps_per_second": 6.257
+    "eval_samples_per_second": 12.714,
+    "eval_steps_per_second": 1.589,
+    "total_flos": 3.803274433029734e+16,
+    "train_loss": 1.5316169482421875,
+    "train_runtime": 15895.0874,
+    "train_samples": 576000,
+    "train_samples_per_second": 100.66,
+    "train_steps_per_second": 6.291
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 0.354862233609357,
-    "eval_bleu": 26.3346,
-    "eval_gen_len": 26.6907,
-    "eval_loss": 1.6253596544265747,
-    "eval_runtime": 221.231,
+    "epoch": 2.7777777777777777,
+    "eval_bleu": 23.6596,
+    "eval_gen_len": 27.526,
+    "eval_loss": 1.7469114065170288,
+    "eval_runtime": 235.9606,
     "eval_samples": 3000,
-    "eval_samples_per_second": 13.56,
-    "eval_steps_per_second": 1.695
+    "eval_samples_per_second": 12.714,
+    "eval_steps_per_second": 1.589
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba60101c5760711223e20065dd911fd73d4bf74287240235d072840395e8aaaa
+oid sha256:e0e0cb2c8f5b00f2545b91fa799ac84d339f1cf8f11cbb40722e1f7b08bf1d74
 size 241984552

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 128,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.354862233609357,
-    "total_flos": 3.589813132276531e+16,
-    "train_loss": 2.132816611328125,
-    "train_runtime": 15981.9131,
-    "train_samples": 4508785,
-    "train_samples_per_second": 100.113,
-    "train_steps_per_second": 6.257
+    "epoch": 2.7777777777777777,
+    "total_flos": 3.803274433029734e+16,
+    "train_loss": 1.5316169482421875,
+    "train_runtime": 15895.0874,
+    "train_samples": 576000,
+    "train_samples_per_second": 100.66,
+    "train_steps_per_second": 6.291
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 26.3346,
+  "best_metric": 23.6596,
   "best_model_checkpoint": "/local1/hfs/gs_stuff/ft-wmt14/checkpoint-100000",
-  "epoch": 0.354862233609357,
+  "epoch": 2.7777777777777777,
   "eval_steps": 10000,
   "global_step": 100000,
   "is_hyper_param_search": false,
@@ -9,261 +9,261 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.01774311168046785,
-      "grad_norm": 1.5128649473190308,
+      "epoch": 0.1388888888888889,
+      "grad_norm": 1.066943645477295,
       "learning_rate": 0.000475,
-      "loss": 2.4524,
+      "loss": 1.9627,
       "step": 5000
     },
     {
-      "epoch": 0.0354862233609357,
-      "grad_norm": 1.242777705192566,
+      "epoch": 0.2777777777777778,
+      "grad_norm": 0.9774492383003235,
       "learning_rate": 0.00045000000000000004,
-      "loss": 2.3103,
+      "loss": 1.7738,
       "step": 10000
     },
     {
-      "epoch": 0.0354862233609357,
-      "eval_bleu": 22.058,
-      "eval_gen_len": 27.7263,
-      "eval_loss": 1.8454290628433228,
-      "eval_runtime": 238.9627,
-      "eval_samples_per_second": 12.554,
-      "eval_steps_per_second": 1.569,
+      "epoch": 0.2777777777777778,
+      "eval_bleu": 20.1598,
+      "eval_gen_len": 28.1563,
+      "eval_loss": 1.914583444595337,
+      "eval_runtime": 241.8013,
+      "eval_samples_per_second": 12.407,
+      "eval_steps_per_second": 1.551,
       "step": 10000
     },
     {
-      "epoch": 0.05322933504140355,
-      "grad_norm": 0.988516628742218,
+      "epoch": 0.4166666666666667,
+      "grad_norm": 1.4306731224060059,
       "learning_rate": 0.000425,
-      "loss": 2.2594,
+      "loss": 1.6951,
       "step": 15000
     },
     {
-      "epoch": 0.0709724467218714,
-      "grad_norm": 1.4387503862380981,
+      "epoch": 0.5555555555555556,
+      "grad_norm": 1.1782424449920654,
       "learning_rate": 0.0004,
-      "loss": 2.2141,
+      "loss": 1.6498,
       "step": 20000
     },
     {
-      "epoch": 0.0709724467218714,
-      "eval_bleu": 23.339,
-      "eval_gen_len": 26.7147,
-      "eval_loss": 1.7811188697814941,
-      "eval_runtime": 225.1917,
-      "eval_samples_per_second": 13.322,
-      "eval_steps_per_second": 1.665,
+      "epoch": 0.5555555555555556,
+      "eval_bleu": 21.4167,
+      "eval_gen_len": 27.853,
+      "eval_loss": 1.855008840560913,
+      "eval_runtime": 242.3949,
+      "eval_samples_per_second": 12.376,
+      "eval_steps_per_second": 1.547,
       "step": 20000
     },
     {
-      "epoch": 0.08871555840233925,
-      "grad_norm": 1.2291666269302368,
+      "epoch": 0.6944444444444444,
+      "grad_norm": 1.219376802444458,
       "learning_rate": 0.000375,
-      "loss": 2.1898,
+      "loss": 1.6172,
       "step": 25000
     },
     {
-      "epoch": 0.1064586700828071,
-      "grad_norm": 1.1732761859893799,
+      "epoch": 0.8333333333333334,
+      "grad_norm": 1.2735612392425537,
       "learning_rate": 0.00035,
-      "loss": 2.176,
+      "loss": 1.5903,
       "step": 30000
     },
     {
-      "epoch": 0.1064586700828071,
-      "eval_bleu": 24.3234,
-      "eval_gen_len": 27.125,
-      "eval_loss": 1.7360602617263794,
-      "eval_runtime": 227.357,
-      "eval_samples_per_second": 13.195,
-      "eval_steps_per_second": 1.649,
+      "epoch": 0.8333333333333334,
+      "eval_bleu": 22.604,
+      "eval_gen_len": 27.7613,
+      "eval_loss": 1.8276705741882324,
+      "eval_runtime": 240.5149,
+      "eval_samples_per_second": 12.473,
+      "eval_steps_per_second": 1.559,
       "step": 30000
     },
     {
-      "epoch": 0.12420178176327495,
-      "grad_norm": 1.187321662902832,
+      "epoch": 0.9722222222222222,
+      "grad_norm": 1.0282609462738037,
       "learning_rate": 0.00032500000000000004,
-      "loss": 2.1468,
+      "loss": 1.5633,
       "step": 35000
     },
     {
-      "epoch": 0.1419448934437428,
-      "grad_norm": 1.3599053621292114,
+      "epoch": 1.1111111111111112,
+      "grad_norm": 1.406827688217163,
       "learning_rate": 0.0003,
-      "loss": 2.139,
+      "loss": 1.5151,
       "step": 40000
     },
     {
-      "epoch": 0.1419448934437428,
-      "eval_bleu": 25.0888,
-      "eval_gen_len": 26.8213,
-      "eval_loss": 1.7130982875823975,
-      "eval_runtime": 221.7983,
-      "eval_samples_per_second": 13.526,
-      "eval_steps_per_second": 1.691,
+      "epoch": 1.1111111111111112,
+      "eval_bleu": 22.1273,
+      "eval_gen_len": 27.3187,
+      "eval_loss": 1.8127936124801636,
+      "eval_runtime": 234.7049,
+      "eval_samples_per_second": 12.782,
+      "eval_steps_per_second": 1.598,
       "step": 40000
     },
     {
-      "epoch": 0.15968800512421066,
-      "grad_norm": 1.4392811059951782,
+      "epoch": 1.25,
+      "grad_norm": 1.174306035041809,
       "learning_rate": 0.000275,
-      "loss": 2.1151,
+      "loss": 1.5004,
       "step": 45000
     },
     {
-      "epoch": 0.1774311168046785,
-      "grad_norm": 1.4162044525146484,
+      "epoch": 1.3888888888888888,
+      "grad_norm": 1.5665515661239624,
       "learning_rate": 0.00025,
-      "loss": 2.1084,
+      "loss": 1.4866,
       "step": 50000
     },
     {
-      "epoch": 0.1774311168046785,
-      "eval_bleu": 24.9992,
-      "eval_gen_len": 26.824,
-      "eval_loss": 1.687427043914795,
-      "eval_runtime": 224.0057,
-      "eval_samples_per_second": 13.393,
-      "eval_steps_per_second": 1.674,
+      "epoch": 1.3888888888888888,
+      "eval_bleu": 22.8295,
+      "eval_gen_len": 27.419,
+      "eval_loss": 1.7999275922775269,
+      "eval_runtime": 233.8115,
+      "eval_samples_per_second": 12.831,
+      "eval_steps_per_second": 1.604,
       "step": 50000
     },
     {
-      "epoch": 0.19517422848514634,
-      "grad_norm": 1.2046048641204834,
+      "epoch": 1.5277777777777777,
+      "grad_norm": 1.1425319910049438,
       "learning_rate": 0.00022500000000000002,
-      "loss": 2.0914,
+      "loss": 1.4799,
       "step": 55000
     },
     {
-      "epoch": 0.2129173401656142,
-      "grad_norm": 1.2651879787445068,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 1.123904824256897,
       "learning_rate": 0.0002,
-      "loss": 2.0826,
+      "loss": 1.4696,
       "step": 60000
     },
     {
-      "epoch": 0.2129173401656142,
-      "eval_bleu": 25.7297,
-      "eval_gen_len": 26.62,
-      "eval_loss": 1.6685482263565063,
-      "eval_runtime": 221.6914,
-      "eval_samples_per_second": 13.532,
-      "eval_steps_per_second": 1.692,
+      "epoch": 1.6666666666666665,
+      "eval_bleu": 22.9923,
+      "eval_gen_len": 27.7387,
+      "eval_loss": 1.780959963798523,
+      "eval_runtime": 240.0938,
+      "eval_samples_per_second": 12.495,
+      "eval_steps_per_second": 1.562,
       "step": 60000
     },
     {
-      "epoch": 0.23066045184608205,
-      "grad_norm": 1.212643027305603,
+      "epoch": 1.8055555555555556,
+      "grad_norm": 1.4292243719100952,
       "learning_rate": 0.000175,
-      "loss": 2.0778,
+      "loss": 1.4613,
       "step": 65000
     },
     {
-      "epoch": 0.2484035635265499,
-      "grad_norm": 1.2400418519973755,
+      "epoch": 1.9444444444444444,
+      "grad_norm": 1.1662226915359497,
       "learning_rate": 0.00015,
-      "loss": 2.068,
+      "loss": 1.4508,
       "step": 70000
     },
     {
-      "epoch": 0.2484035635265499,
-      "eval_bleu": 25.9031,
-      "eval_gen_len": 26.685,
-      "eval_loss": 1.648539662361145,
-      "eval_runtime": 223.211,
-      "eval_samples_per_second": 13.44,
-      "eval_steps_per_second": 1.68,
+      "epoch": 1.9444444444444444,
+      "eval_bleu": 23.1046,
+      "eval_gen_len": 27.7057,
+      "eval_loss": 1.7654317617416382,
+      "eval_runtime": 236.6367,
+      "eval_samples_per_second": 12.678,
+      "eval_steps_per_second": 1.585,
       "step": 70000
     },
     {
-      "epoch": 0.26614667520701774,
-      "grad_norm": 1.3389995098114014,
+      "epoch": 2.0833333333333335,
+      "grad_norm": 0.9245423674583435,
       "learning_rate": 0.000125,
-      "loss": 2.0566,
+      "loss": 1.4235,
       "step": 75000
     },
     {
-      "epoch": 0.2838897868874856,
-      "grad_norm": 1.1512677669525146,
+      "epoch": 2.2222222222222223,
+      "grad_norm": 1.2502944469451904,
       "learning_rate": 0.0001,
-      "loss": 2.05,
+      "loss": 1.4053,
       "step": 80000
     },
     {
-      "epoch": 0.2838897868874856,
-      "eval_bleu": 26.143,
-      "eval_gen_len": 26.8693,
-      "eval_loss": 1.6370748281478882,
-      "eval_runtime": 225.2245,
-      "eval_samples_per_second": 13.32,
-      "eval_steps_per_second": 1.665,
+      "epoch": 2.2222222222222223,
+      "eval_bleu": 23.5079,
+      "eval_gen_len": 27.643,
+      "eval_loss": 1.758699655532837,
+      "eval_runtime": 237.5663,
+      "eval_samples_per_second": 12.628,
+      "eval_steps_per_second": 1.579,
       "step": 80000
     },
     {
-      "epoch": 0.30163289856795344,
-      "grad_norm": 1.1607016324996948,
+      "epoch": 2.361111111111111,
+      "grad_norm": 0.9593023061752319,
       "learning_rate": 7.5e-05,
-      "loss": 2.0235,
+      "loss": 1.408,
       "step": 85000
     },
     {
-      "epoch": 0.3193760102484213,
-      "grad_norm": 1.2967106103897095,
+      "epoch": 2.5,
+      "grad_norm": 1.440004825592041,
       "learning_rate": 5e-05,
-      "loss": 2.0331,
+      "loss": 1.3956,
       "step": 90000
     },
     {
-      "epoch": 0.3193760102484213,
-      "eval_bleu": 26.3038,
-      "eval_gen_len": 26.5183,
-      "eval_loss": 1.6311123371124268,
-      "eval_runtime": 219.2546,
-      "eval_samples_per_second": 13.683,
-      "eval_steps_per_second": 1.71,
+      "epoch": 2.5,
+      "eval_bleu": 23.3848,
+      "eval_gen_len": 27.6637,
+      "eval_loss": 1.752461552619934,
+      "eval_runtime": 237.0184,
+      "eval_samples_per_second": 12.657,
+      "eval_steps_per_second": 1.582,
       "step": 90000
     },
     {
-      "epoch": 0.33711912192888915,
-      "grad_norm": 1.2956724166870117,
+      "epoch": 2.638888888888889,
+      "grad_norm": 1.1929932832717896,
       "learning_rate": 2.5e-05,
-      "loss": 2.0346,
+      "loss": 1.3938,
       "step": 95000
     },
     {
-      "epoch": 0.354862233609357,
-      "grad_norm": 1.1822398900985718,
+      "epoch": 2.7777777777777777,
+      "grad_norm": 1.0216492414474487,
       "learning_rate": 0.0,
-      "loss": 2.0273,
+      "loss": 1.3903,
       "step": 100000
     },
     {
-      "epoch": 0.354862233609357,
-      "eval_bleu": 26.3346,
-      "eval_gen_len": 26.6907,
-      "eval_loss": 1.6253596544265747,
-      "eval_runtime": 221.5579,
-      "eval_samples_per_second": 13.54,
-      "eval_steps_per_second": 1.693,
+      "epoch": 2.7777777777777777,
+      "eval_bleu": 23.6596,
+      "eval_gen_len": 27.526,
+      "eval_loss": 1.7469114065170288,
+      "eval_runtime": 235.9542,
+      "eval_samples_per_second": 12.714,
+      "eval_steps_per_second": 1.589,
       "step": 100000
     },
     {
-      "epoch": 0.354862233609357,
+      "epoch": 2.7777777777777777,
       "step": 100000,
-      "total_flos": 3.589813132276531e+16,
-      "train_loss": 2.132816611328125,
-      "train_runtime": 15981.9131,
-      "train_samples_per_second": 100.113,
-      "train_steps_per_second": 6.257
+      "total_flos": 3.803274433029734e+16,
+      "train_loss": 1.5316169482421875,
+      "train_runtime": 15895.0874,
+      "train_samples_per_second": 100.66,
+      "train_steps_per_second": 6.291
     }
   ],
   "logging_steps": 5000,
   "max_steps": 100000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 3,
   "save_steps": 10000,
-  "total_flos": 3.589813132276531e+16,
+  "total_flos": 3.803274433029734e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c032e73464a2058a913f4433764c71c06f52d322360e96ce929b33d49f129624
-size 5176
+oid sha256:cca3b2463e1aca65e73c3842cf3e0162ea69920dc5160ae31c9b16756f63072a
+size 5112