Model save

README.md

@@ -17,10 +17,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 3.1985
 - Accuracy: 0.4196
-- Perplexity: 24.4954
 - Bleu: 0.1339
+- Loss: 3.1985
+- Perplexity: 24.4954
 
 ## Model description
 
@@ -50,25 +50,25 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Accuracy | Perplexity | Bleu   |
-|:-------------:|:------:|:----:|:---------------:|:--------:|:----------:|:------:|
-| 5.9062        | 0.2806 | 500  | 5.7470          | 0.2234   | 313.2463   | 0.0493 |
-| 4.8598        | 0.5612 | 1000 | 4.7428          | 0.2811   | 114.7554   | 0.0698 |
-| 4.3025        | 0.8418 | 1500 | 4.2329          | 0.3170   | 68.9191    | 0.0834 |
-| 3.9635        | 1.1223 | 2000 | 3.9291          | 0.3454   | 50.8590    | 0.0932 |
-| 3.7769        | 1.4029 | 2500 | 3.7427          | 0.3636   | 42.2098    | 0.1020 |
-| 3.6738        | 1.6835 | 3000 | 3.6225          | 0.3754   | 37.4295    | 0.1066 |
-| 3.5744        | 1.9641 | 3500 | 3.5325          | 0.3845   | 34.2102    | 0.1118 |
-| 3.456         | 2.2447 | 4000 | 3.4704          | 0.3902   | 32.1497    | 0.1139 |
-| 3.3972        | 2.5253 | 4500 | 3.4190          | 0.3955   | 30.5384    | 0.1230 |
-| 3.3654        | 2.8058 | 5000 | 3.3686          | 0.4007   | 29.0392    | 0.1230 |
-| 3.247         | 3.0864 | 5500 | 3.3328          | 0.4043   | 28.0168    | 0.1247 |
-| 3.2403        | 3.3670 | 6000 | 3.2985          | 0.4083   | 27.0714    | 0.1298 |
-| 3.2167        | 3.6476 | 6500 | 3.2693          | 0.4112   | 26.2922    | 0.1288 |
-| 3.1903        | 3.9282 | 7000 | 3.2456          | 0.4134   | 25.6768    | 0.1305 |
-| 3.1212        | 4.2088 | 7500 | 3.2262          | 0.4161   | 25.1831    | 0.1325 |
-| 3.0816        | 4.4893 | 8000 | 3.2128          | 0.4176   | 24.8480    | 0.1307 |
-| 3.0917        | 4.7699 | 8500 | 3.1985          | 0.4196   | 24.4954    | 0.1339 |
+| Training Loss | Epoch  | Step | Accuracy | Bleu   | Validation Loss | Perplexity |
+|:-------------:|:------:|:----:|:--------:|:------:|:---------------:|:----------:|
+| 5.9062        | 0.2806 | 500  | 0.2234   | 0.0493 | 5.7470          | 313.2463   |
+| 4.8598        | 0.5612 | 1000 | 0.2811   | 0.0698 | 4.7428          | 114.7554   |
+| 4.3025        | 0.8418 | 1500 | 0.3170   | 0.0834 | 4.2329          | 68.9191    |
+| 3.9635        | 1.1223 | 2000 | 0.3454   | 0.0932 | 3.9291          | 50.8590    |
+| 3.7769        | 1.4029 | 2500 | 0.3636   | 0.1020 | 3.7427          | 42.2098    |
+| 3.6738        | 1.6835 | 3000 | 0.3754   | 0.1066 | 3.6225          | 37.4295    |
+| 3.5744        | 1.9641 | 3500 | 0.3845   | 0.1118 | 3.5325          | 34.2102    |
+| 3.456         | 2.2447 | 4000 | 0.3902   | 0.1139 | 3.4704          | 32.1497    |
+| 3.3972        | 2.5253 | 4500 | 0.3955   | 0.1230 | 3.4190          | 30.5384    |
+| 3.3654        | 2.8058 | 5000 | 0.4007   | 0.1230 | 3.3686          | 29.0392    |
+| 3.247         | 3.0864 | 5500 | 0.4043   | 0.1247 | 3.3328          | 28.0168    |
+| 3.2403        | 3.3670 | 6000 | 0.4083   | 0.1298 | 3.2985          | 27.0714    |
+| 3.2167        | 3.6476 | 6500 | 0.4112   | 0.1288 | 3.2693          | 26.2922    |
+| 3.1903        | 3.9282 | 7000 | 0.4134   | 0.1305 | 3.2456          | 25.6768    |
+| 3.1212        | 4.2088 | 7500 | 0.4161   | 0.1325 | 3.2262          | 25.1831    |
+| 3.0816        | 4.4893 | 8000 | 0.4176   | 0.1307 | 3.2128          | 24.8480    |
+| 3.0917        | 4.7699 | 8500 | 0.4196   | 0.1339 | 3.1985          | 24.4954    |
 
 
 ### Framework versions

all_results.json

@@ -1,17 +1,17 @@
 {
     "epoch": 5.0,
-    "eval_accuracy": 0.4192811608208402,
-    "eval_bleu": 0.13449590326973765,
-    "eval_loss": 3.201490640640259,
-    "eval_perplexity": 24.569126652935235,
-    "eval_runtime": 12.2552,
+    "eval_accuracy": 0.41997167684877956,
+    "eval_bleu": 0.13436705184218095,
+    "eval_loss": 3.19417405128479,
+    "eval_perplexity": 24.39002046460787,
+    "eval_runtime": 12.1627,
     "eval_samples": 1141,
-    "eval_samples_per_second": 93.103,
-    "eval_steps_per_second": 1.469,
-    "perplexity": 24.569126652935235,
+    "eval_samples_per_second": 93.812,
+    "eval_steps_per_second": 1.48,
+    "perplexity": 24.39002046460787,
     "total_flos": 1.0587061010143642e+18,
-    "train_loss": 3.7750147520893753,
-    "train_runtime": 15697.9119,
-    "train_samples_per_second": 36.309,
-    "train_steps_per_second": 0.568
+    "train_loss": 3.767289323945907,
+    "train_runtime": 15738.6672,
+    "train_samples_per_second": 36.215,
+    "train_steps_per_second": 0.566
 }

eval_results.json

@@ -1,12 +1,12 @@
 {
     "epoch": 5.0,
-    "eval_accuracy": 0.4192811608208402,
-    "eval_bleu": 0.13449590326973765,
-    "eval_loss": 3.201490640640259,
-    "eval_perplexity": 24.569126652935235,
-    "eval_runtime": 12.2552,
+    "eval_accuracy": 0.41997167684877956,
+    "eval_bleu": 0.13436705184218095,
+    "eval_loss": 3.19417405128479,
+    "eval_perplexity": 24.39002046460787,
+    "eval_runtime": 12.1627,
     "eval_samples": 1141,
-    "eval_samples_per_second": 93.103,
-    "eval_steps_per_second": 1.469,
-    "perplexity": 24.569126652935235
+    "eval_samples_per_second": 93.812,
+    "eval_steps_per_second": 1.48,
+    "perplexity": 24.39002046460787
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 5.0,
     "total_flos": 1.0587061010143642e+18,
-    "train_loss": 3.7750147520893753,
-    "train_runtime": 15697.9119,
-    "train_samples_per_second": 36.309,
-    "train_steps_per_second": 0.568
+    "train_loss": 3.767289323945907,
+    "train_runtime": 15738.6672,
+    "train_samples_per_second": 36.215,
+    "train_steps_per_second": 0.566
 }

trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 3.2064151763916016,
+  "best_metric": 3.1984846591949463,
   "best_model_checkpoint": "./output/models/rotating-head-gp-gpt2-medium-wikitext/checkpoint-8500",
   "epoch": 5.0,
   "eval_steps": 500,
@@ -10,822 +10,822 @@
   "log_history": [
     {
       "epoch": 0.05611672278338945,
-      "grad_norm": 1.7030696868896484,
+      "grad_norm": 1.7026984691619873,
       "learning_rate": 1.1223344556677892e-05,
       "loss": 8.993,
       "step": 100
     },
     {
       "epoch": 0.1122334455667789,
-      "grad_norm": 1.4508291482925415,
+      "grad_norm": 1.4564013481140137,
       "learning_rate": 2.2446689113355783e-05,
       "loss": 7.3838,
       "step": 200
     },
     {
       "epoch": 0.16835016835016836,
-      "grad_norm": 1.6375716924667358,
+      "grad_norm": 1.6578171253204346,
       "learning_rate": 3.3670033670033675e-05,
-      "loss": 6.5795,
+      "loss": 6.5796,
       "step": 300
     },
     {
       "epoch": 0.2244668911335578,
-      "grad_norm": 1.417483925819397,
+      "grad_norm": 2.0035881996154785,
       "learning_rate": 4.4893378226711566e-05,
-      "loss": 6.1961,
+      "loss": 6.1995,
       "step": 400
     },
     {
       "epoch": 0.28058361391694725,
-      "grad_norm": 1.7931348085403442,
+      "grad_norm": 1.3597742319107056,
       "learning_rate": 5.611672278338945e-05,
-      "loss": 5.9051,
+      "loss": 5.9062,
       "step": 500
     },
     {
       "epoch": 0.28058361391694725,
-      "eval_accuracy": 0.2238085814179224,
-      "eval_bleu": 0.04813423949717817,
-      "eval_loss": 5.7417893409729,
-      "eval_perplexity": 311.6215095481783,
-      "eval_runtime": 12.102,
-      "eval_samples_per_second": 94.282,
-      "eval_steps_per_second": 1.487,
+      "eval_accuracy": 0.22335966032779805,
+      "eval_bleu": 0.0492964548196352,
+      "eval_loss": 5.746989727020264,
+      "eval_perplexity": 313.2462827607658,
+      "eval_runtime": 12.1113,
+      "eval_samples_per_second": 94.209,
+      "eval_steps_per_second": 1.486,
       "step": 500
     },
     {
       "epoch": 0.3367003367003367,
-      "grad_norm": 1.502821445465088,
+      "grad_norm": 1.2420822381973267,
       "learning_rate": 6.734006734006735e-05,
-      "loss": 5.6571,
+      "loss": 5.6583,
       "step": 600
     },
     {
       "epoch": 0.39281705948372614,
-      "grad_norm": 1.267421007156372,
+      "grad_norm": 1.7358133792877197,
       "learning_rate": 7.856341189674523e-05,
-      "loss": 5.4352,
+      "loss": 5.4354,
       "step": 700
     },
     {
       "epoch": 0.4489337822671156,
-      "grad_norm": 1.4694616794586182,
+      "grad_norm": 1.1559091806411743,
       "learning_rate": 8.978675645342313e-05,
-      "loss": 5.2521,
+      "loss": 5.252,
       "step": 800
     },
     {
       "epoch": 0.5050505050505051,
-      "grad_norm": 1.2315590381622314,
+      "grad_norm": 1.1872116327285767,
       "learning_rate": 9.988776655443322e-05,
-      "loss": 5.0471,
+      "loss": 5.0452,
       "step": 900
     },
     {
       "epoch": 0.5611672278338945,
-      "grad_norm": 0.9069780707359314,
+      "grad_norm": 1.0503571033477783,
       "learning_rate": 9.864072827035791e-05,
-      "loss": 4.8617,
+      "loss": 4.8598,
       "step": 1000
     },
     {
       "epoch": 0.5611672278338945,
-      "eval_accuracy": 0.28116767459732034,
-      "eval_bleu": 0.07213900102284179,
-      "eval_loss": 4.740506649017334,
-      "eval_perplexity": 114.4921943457123,
-      "eval_runtime": 12.1272,
-      "eval_samples_per_second": 94.086,
-      "eval_steps_per_second": 1.484,
+      "eval_accuracy": 0.28109999374594663,
+      "eval_bleu": 0.06981793198064049,
+      "eval_loss": 4.74280309677124,
+      "eval_perplexity": 114.75542181664323,
+      "eval_runtime": 12.1357,
+      "eval_samples_per_second": 94.02,
+      "eval_steps_per_second": 1.483,
       "step": 1000
     },
     {
       "epoch": 0.6172839506172839,
-      "grad_norm": 0.9436570405960083,
+      "grad_norm": 0.9331828951835632,
       "learning_rate": 9.73936899862826e-05,
-      "loss": 4.7137,
+      "loss": 4.7116,
       "step": 1100
     },
     {
       "epoch": 0.6734006734006734,
-      "grad_norm": 0.8758283853530884,
+      "grad_norm": 0.8568651676177979,
       "learning_rate": 9.614665170220725e-05,
-      "loss": 4.5873,
+      "loss": 4.5871,
       "step": 1200
     },
     {
       "epoch": 0.7295173961840629,
-      "grad_norm": 0.8786563277244568,
+      "grad_norm": 0.7943041920661926,
       "learning_rate": 9.489961341813194e-05,
-      "loss": 4.4798,
+      "loss": 4.48,
       "step": 1300
     },
     {
       "epoch": 0.7856341189674523,
-      "grad_norm": 0.9080842733383179,
+      "grad_norm": 0.929481029510498,
       "learning_rate": 9.365257513405662e-05,
-      "loss": 4.3915,
+      "loss": 4.3936,
       "step": 1400
     },
     {
       "epoch": 0.8417508417508418,
-      "grad_norm": 0.9573113322257996,
+      "grad_norm": 0.9010036587715149,
       "learning_rate": 9.24055368499813e-05,
-      "loss": 4.2992,
+      "loss": 4.3025,
       "step": 1500
     },
     {
       "epoch": 0.8417508417508418,
-      "eval_accuracy": 0.317916663453968,
-      "eval_bleu": 0.08316362600465955,
-      "eval_loss": 4.227725982666016,
-      "eval_perplexity": 68.56114553586224,
-      "eval_runtime": 12.2167,
-      "eval_samples_per_second": 93.397,
-      "eval_steps_per_second": 1.473,
+      "eval_accuracy": 0.31702567503082046,
+      "eval_bleu": 0.08337643356737867,
+      "eval_loss": 4.23293399810791,
+      "eval_perplexity": 68.91914446331879,
+      "eval_runtime": 12.1629,
+      "eval_samples_per_second": 93.809,
+      "eval_steps_per_second": 1.48,
       "step": 1500
     },
     {
       "epoch": 0.8978675645342312,
-      "grad_norm": 1.070070505142212,
+      "grad_norm": 0.8791123628616333,
       "learning_rate": 9.115849856590598e-05,
-      "loss": 4.2381,
+      "loss": 4.2455,
       "step": 1600
     },
     {
       "epoch": 0.9539842873176206,
-      "grad_norm": 1.0821105241775513,
+      "grad_norm": 0.938328742980957,
       "learning_rate": 8.991146028183066e-05,
-      "loss": 4.1477,
+      "loss": 4.1605,
       "step": 1700
     },
     {
       "epoch": 1.0101010101010102,
-      "grad_norm": 1.0546807050704956,
+      "grad_norm": 1.0388487577438354,
       "learning_rate": 8.866442199775533e-05,
-      "loss": 4.0864,
+      "loss": 4.1011,
       "step": 1800
     },
     {
       "epoch": 1.0662177328843996,
-      "grad_norm": 1.0770821571350098,
+      "grad_norm": 0.9663709998130798,
       "learning_rate": 8.741738371368002e-05,
-      "loss": 4.0053,
+      "loss": 4.0148,
       "step": 1900
     },
     {
       "epoch": 1.122334455667789,
-      "grad_norm": 0.9692613482475281,
+      "grad_norm": 0.8401734828948975,
       "learning_rate": 8.617034542960469e-05,
-      "loss": 3.9585,
+      "loss": 3.9635,
       "step": 2000
     },
     {
       "epoch": 1.122334455667789,
-      "eval_accuracy": 0.34663219226844794,
-      "eval_bleu": 0.09053001364011155,
-      "eval_loss": 3.9252164363861084,
-      "eval_perplexity": 50.66404241449891,
-      "eval_runtime": 12.3364,
-      "eval_samples_per_second": 92.491,
-      "eval_steps_per_second": 1.459,
+      "eval_accuracy": 0.34535482328872397,
+      "eval_bleu": 0.09320946829041966,
+      "eval_loss": 3.9290566444396973,
+      "eval_perplexity": 50.85897693319932,
+      "eval_runtime": 12.2142,
+      "eval_samples_per_second": 93.416,
+      "eval_steps_per_second": 1.474,
       "step": 2000
     },
     {
       "epoch": 1.1784511784511784,
-      "grad_norm": 0.9078754782676697,
+      "grad_norm": 1.1741111278533936,
       "learning_rate": 8.492330714552937e-05,
-      "loss": 3.9194,
+      "loss": 3.9219,
       "step": 2100
     },
     {
       "epoch": 1.2345679012345678,
-      "grad_norm": 1.207867980003357,
+      "grad_norm": 1.1291131973266602,
       "learning_rate": 8.367626886145406e-05,
-      "loss": 3.8743,
+      "loss": 3.876,
       "step": 2200
     },
     {
       "epoch": 1.2906846240179575,
-      "grad_norm": 1.2701904773712158,
+      "grad_norm": 1.0407253503799438,
       "learning_rate": 8.242923057737873e-05,
-      "loss": 3.8596,
+      "loss": 3.8557,
       "step": 2300
     },
     {
       "epoch": 1.3468013468013469,
-      "grad_norm": 1.4610146284103394,
+      "grad_norm": 1.0766143798828125,
       "learning_rate": 8.11821922933034e-05,
-      "loss": 3.821,
+      "loss": 3.8141,
       "step": 2400
     },
     {
       "epoch": 1.4029180695847363,
-      "grad_norm": 2.153532028198242,
+      "grad_norm": 1.42284095287323,
       "learning_rate": 7.993515400922809e-05,
-      "loss": 3.7838,
+      "loss": 3.7769,
       "step": 2500
     },
     {
       "epoch": 1.4029180695847363,
-      "eval_accuracy": 0.36260658663191814,
-      "eval_bleu": 0.09858369505966838,
-      "eval_loss": 3.7564430236816406,
-      "eval_perplexity": 42.7959308162683,
-      "eval_runtime": 12.2808,
-      "eval_samples_per_second": 92.909,
-      "eval_steps_per_second": 1.466,
+      "eval_accuracy": 0.36358581717774274,
+      "eval_bleu": 0.10197996736893164,
+      "eval_loss": 3.7426531314849854,
+      "eval_perplexity": 42.209829965401774,
+      "eval_runtime": 12.1768,
+      "eval_samples_per_second": 93.703,
+      "eval_steps_per_second": 1.478,
       "step": 2500
     },
     {
       "epoch": 1.4590347923681257,
-      "grad_norm": 1.3955634832382202,
+      "grad_norm": 1.3685775995254517,
       "learning_rate": 7.868811572515277e-05,
-      "loss": 3.7668,
+      "loss": 3.7592,
       "step": 2600
     },
     {
       "epoch": 1.5151515151515151,
-      "grad_norm": 1.3527532815933228,
+      "grad_norm": 1.6901907920837402,
       "learning_rate": 7.744107744107744e-05,
-      "loss": 3.7424,
+      "loss": 3.7311,
       "step": 2700
     },
     {
       "epoch": 1.5712682379349046,
-      "grad_norm": 0.9464800357818604,
+      "grad_norm": 1.2826731204986572,
       "learning_rate": 7.619403915700213e-05,
-      "loss": 3.708,
+      "loss": 3.6989,
       "step": 2800
     },
     {
       "epoch": 1.627384960718294,
-      "grad_norm": 1.9855220317840576,
+      "grad_norm": 1.1707065105438232,
       "learning_rate": 7.49470008729268e-05,
-      "loss": 3.6981,
+      "loss": 3.6858,
       "step": 2900
     },
     {
       "epoch": 1.6835016835016834,
-      "grad_norm": 2.4134933948516846,
+      "grad_norm": 1.8538202047348022,
       "learning_rate": 7.369996258885148e-05,
-      "loss": 3.6863,
+      "loss": 3.6738,
       "step": 3000
     },
     {
       "epoch": 1.6835016835016834,
-      "eval_accuracy": 0.3738518885956052,
-      "eval_bleu": 0.10686414409488354,
-      "eval_loss": 3.6387925148010254,
-      "eval_perplexity": 38.04586915451721,
-      "eval_runtime": 12.2104,
-      "eval_samples_per_second": 93.445,
-      "eval_steps_per_second": 1.474,
+      "eval_accuracy": 0.3753888436255347,
+      "eval_bleu": 0.1065937514211335,
+      "eval_loss": 3.6224589347839355,
+      "eval_perplexity": 37.429491437089155,
+      "eval_runtime": 12.1708,
+      "eval_samples_per_second": 93.749,
+      "eval_steps_per_second": 1.479,
       "step": 3000
     },
     {
       "epoch": 1.7396184062850728,
-      "grad_norm": 2.7664296627044678,
+      "grad_norm": 1.2428492307662964,
       "learning_rate": 7.245292430477615e-05,
-      "loss": 3.6503,
+      "loss": 3.6346,
       "step": 3100
     },
     {
       "epoch": 1.7957351290684624,
-      "grad_norm": 2.4914608001708984,
+      "grad_norm": 1.5533177852630615,
       "learning_rate": 7.120588602070084e-05,
-      "loss": 3.6382,
+      "loss": 3.6228,
       "step": 3200
     },
     {
       "epoch": 1.8518518518518519,
-      "grad_norm": 2.2810556888580322,
+      "grad_norm": 1.538769006729126,
       "learning_rate": 6.995884773662552e-05,
-      "loss": 3.6126,
+      "loss": 3.5964,
       "step": 3300
     },
     {
       "epoch": 1.9079685746352413,
-      "grad_norm": 1.139969825744629,
+      "grad_norm": 0.9837027788162231,
       "learning_rate": 6.871180945255021e-05,
-      "loss": 3.6042,
+      "loss": 3.5916,
       "step": 3400
     },
     {
       "epoch": 1.964085297418631,
-      "grad_norm": 1.3077082633972168,
+      "grad_norm": 1.3107187747955322,
       "learning_rate": 6.746477116847487e-05,
-      "loss": 3.5869,
+      "loss": 3.5744,
       "step": 3500
     },
     {
       "epoch": 1.964085297418631,
-      "eval_accuracy": 0.3825578735533218,
-      "eval_bleu": 0.11004806187463387,
-      "eval_loss": 3.5517890453338623,
-      "eval_perplexity": 34.875655837249155,
-      "eval_runtime": 12.1547,
-      "eval_samples_per_second": 93.873,
-      "eval_steps_per_second": 1.481,
+      "eval_accuracy": 0.38449063305584186,
+      "eval_bleu": 0.11184212819864767,
+      "eval_loss": 3.5325236320495605,
+      "eval_perplexity": 34.21019270752698,
+      "eval_runtime": 12.1937,
+      "eval_samples_per_second": 93.573,
+      "eval_steps_per_second": 1.476,
       "step": 3500
     },
     {
       "epoch": 2.0202020202020203,
-      "grad_norm": 2.2133948802948,
+      "grad_norm": 1.3989722728729248,
       "learning_rate": 6.621773288439955e-05,
-      "loss": 3.5521,
+      "loss": 3.5381,
       "step": 3600
     },
     {
       "epoch": 2.0763187429854097,
-      "grad_norm": 1.2750334739685059,
+      "grad_norm": 1.5801029205322266,
       "learning_rate": 6.497069460032424e-05,
-      "loss": 3.4978,
+      "loss": 3.4865,
       "step": 3700
     },
     {
       "epoch": 2.132435465768799,
-      "grad_norm": 1.4778505563735962,
+      "grad_norm": 1.2122889757156372,
       "learning_rate": 6.372365631624892e-05,
-      "loss": 3.4785,
+      "loss": 3.4665,
       "step": 3800
     },
     {
       "epoch": 2.1885521885521886,
-      "grad_norm": 1.5821393728256226,
+      "grad_norm": 2.2837603092193604,
       "learning_rate": 6.247661803217359e-05,
-      "loss": 3.4804,
+      "loss": 3.4675,
       "step": 3900
     },
     {
       "epoch": 2.244668911335578,
-      "grad_norm": 1.2070300579071045,
+      "grad_norm": 1.1615939140319824,
       "learning_rate": 6.122957974809826e-05,
-      "loss": 3.4733,
+      "loss": 3.456,
       "step": 4000
     },
     {
       "epoch": 2.244668911335578,
-      "eval_accuracy": 0.38860631419507335,
-      "eval_bleu": 0.11590933164079384,
-      "eval_loss": 3.4845945835113525,
-      "eval_perplexity": 32.60920412059944,
-      "eval_runtime": 12.166,
-      "eval_samples_per_second": 93.786,
-      "eval_steps_per_second": 1.48,
+      "eval_accuracy": 0.3902143769549271,
+      "eval_bleu": 0.11388067737811994,
+      "eval_loss": 3.4704020023345947,
+      "eval_perplexity": 32.149664087333434,
+      "eval_runtime": 12.121,
+      "eval_samples_per_second": 94.134,
+      "eval_steps_per_second": 1.485,
       "step": 4000
     },
     {
       "epoch": 2.3007856341189674,
-      "grad_norm": 1.0372384786605835,
+      "grad_norm": 0.9502741694450378,
       "learning_rate": 5.998254146402295e-05,
-      "loss": 3.4604,
+      "loss": 3.4467,
       "step": 4100
     },
     {
       "epoch": 2.356902356902357,
-      "grad_norm": 1.3492435216903687,
+      "grad_norm": 1.7536747455596924,
       "learning_rate": 5.8735503179947625e-05,
-      "loss": 3.4395,
+      "loss": 3.4298,
       "step": 4200
     },
     {
       "epoch": 2.4130190796857462,
-      "grad_norm": 1.0056568384170532,
+      "grad_norm": 1.4674713611602783,
       "learning_rate": 5.748846489587231e-05,
-      "loss": 3.4359,
+      "loss": 3.4221,
       "step": 4300
     },
     {
       "epoch": 2.4691358024691357,
-      "grad_norm": 2.51788067817688,
+      "grad_norm": 2.0394678115844727,
       "learning_rate": 5.624142661179699e-05,
-      "loss": 3.4198,
+      "loss": 3.4072,
       "step": 4400
     },
     {
       "epoch": 2.525252525252525,
-      "grad_norm": 2.3806328773498535,
+      "grad_norm": 2.8717079162597656,
       "learning_rate": 5.4994388327721666e-05,
-      "loss": 3.4122,
+      "loss": 3.3972,
       "step": 4500
     },
     {
       "epoch": 2.525252525252525,
-      "eval_accuracy": 0.39410559754909646,
-      "eval_bleu": 0.12123148173651561,
-      "eval_loss": 3.430687665939331,
-      "eval_perplexity": 30.897882867432642,
-      "eval_runtime": 12.1642,
-      "eval_samples_per_second": 93.8,
-      "eval_steps_per_second": 1.48,
+      "eval_accuracy": 0.3955106177548291,
+      "eval_bleu": 0.12298741482321522,
+      "eval_loss": 3.4189839363098145,
+      "eval_perplexity": 30.53837032278329,
+      "eval_runtime": 12.1708,
+      "eval_samples_per_second": 93.749,
+      "eval_steps_per_second": 1.479,
       "step": 4500
     },
     {
       "epoch": 2.581369248035915,
-      "grad_norm": 1.4060360193252563,
+      "grad_norm": 2.2189624309539795,
       "learning_rate": 5.374735004364634e-05,
-      "loss": 3.4137,
+      "loss": 3.4006,
       "step": 4600
     },
     {
       "epoch": 2.637485970819304,
-      "grad_norm": 1.47615647315979,
+      "grad_norm": 1.444754719734192,
       "learning_rate": 5.250031175957102e-05,
-      "loss": 3.3987,
+      "loss": 3.3886,
       "step": 4700
     },
     {
       "epoch": 2.6936026936026938,
-      "grad_norm": 3.111424446105957,
+      "grad_norm": 1.8333204984664917,
       "learning_rate": 5.12532734754957e-05,
-      "loss": 3.3934,
+      "loss": 3.3813,
       "step": 4800
     },
     {
       "epoch": 2.749719416386083,
-      "grad_norm": 1.385186791419983,
+      "grad_norm": 2.1033811569213867,
       "learning_rate": 5.000623519142038e-05,
-      "loss": 3.3842,
+      "loss": 3.372,
       "step": 4900
     },
     {
       "epoch": 2.8058361391694726,
-      "grad_norm": 1.4257937669754028,
+      "grad_norm": 1.8956849575042725,
       "learning_rate": 4.8759196907345056e-05,
-      "loss": 3.3791,
+      "loss": 3.3654,
       "step": 5000
     },
     {
       "epoch": 2.8058361391694726,
-      "eval_accuracy": 0.39906000721357937,
-      "eval_bleu": 0.12230826630862061,
-      "eval_loss": 3.380352735519409,
-      "eval_perplexity": 29.38113305489852,
-      "eval_runtime": 12.1813,
-      "eval_samples_per_second": 93.668,
-      "eval_steps_per_second": 1.478,
+      "eval_accuracy": 0.40071690299277873,
+      "eval_bleu": 0.12304297750670024,
+      "eval_loss": 3.368644952774048,
+      "eval_perplexity": 29.039150964630583,
+      "eval_runtime": 12.2072,
+      "eval_samples_per_second": 93.469,
+      "eval_steps_per_second": 1.475,
       "step": 5000
     },
     {
       "epoch": 2.861952861952862,
-      "grad_norm": 2.4099814891815186,
+      "grad_norm": 2.0555260181427,
       "learning_rate": 4.751215862326974e-05,
-      "loss": 3.3747,
+      "loss": 3.3622,
       "step": 5100
     },
     {
       "epoch": 2.9180695847362514,
-      "grad_norm": 2.5423948764801025,
+      "grad_norm": 1.1492657661437988,
       "learning_rate": 4.626512033919442e-05,
-      "loss": 3.3545,
+      "loss": 3.3413,
       "step": 5200
     },
     {
       "epoch": 2.974186307519641,
-      "grad_norm": 2.864335775375366,
+      "grad_norm": 2.6185925006866455,
       "learning_rate": 4.5018082055119096e-05,
-      "loss": 3.3586,
+      "loss": 3.3452,
       "step": 5300
     },
     {
       "epoch": 3.0303030303030303,
-      "grad_norm": 2.2575385570526123,
+      "grad_norm": 1.4890856742858887,
       "learning_rate": 4.3771043771043774e-05,
-      "loss": 3.3028,
+      "loss": 3.2908,
       "step": 5400
     },
     {
       "epoch": 3.0864197530864197,
-      "grad_norm": 2.1973252296447754,
+      "grad_norm": 2.316535711288452,
       "learning_rate": 4.252400548696845e-05,
-      "loss": 3.2616,
+      "loss": 3.247,
       "step": 5500
     },
     {
       "epoch": 3.0864197530864197,
-      "eval_accuracy": 0.40264623561674817,
-      "eval_bleu": 0.12218084899348862,
-      "eval_loss": 3.3446905612945557,
-      "eval_perplexity": 28.35180116448892,
-      "eval_runtime": 12.1937,
-      "eval_samples_per_second": 93.573,
-      "eval_steps_per_second": 1.476,
+      "eval_accuracy": 0.40426800589080425,
+      "eval_bleu": 0.1247222065482489,
+      "eval_loss": 3.3328051567077637,
+      "eval_perplexity": 28.016823154790686,
+      "eval_runtime": 12.2011,
+      "eval_samples_per_second": 93.516,
+      "eval_steps_per_second": 1.475,
       "step": 5500
     },
     {
       "epoch": 3.142536475869809,
-      "grad_norm": 1.4363641738891602,
+      "grad_norm": 1.5401073694229126,
       "learning_rate": 4.127696720289313e-05,
-      "loss": 3.2647,
+      "loss": 3.253,
       "step": 5600
     },
     {
       "epoch": 3.1986531986531985,
-      "grad_norm": 1.4158735275268555,
+      "grad_norm": 1.508957028388977,
       "learning_rate": 4.002992891881781e-05,
-      "loss": 3.2614,
+      "loss": 3.2477,
       "step": 5700
     },
     {
       "epoch": 3.254769921436588,
-      "grad_norm": 1.5934867858886719,
+      "grad_norm": 1.5511479377746582,
       "learning_rate": 3.8782890634742486e-05,
-      "loss": 3.2573,
+      "loss": 3.2456,
       "step": 5800
     },
     {
       "epoch": 3.3108866442199774,
-      "grad_norm": 1.3518158197402954,
+      "grad_norm": 1.5875085592269897,
       "learning_rate": 3.7535852350667164e-05,
-      "loss": 3.2584,
+      "loss": 3.2472,
       "step": 5900
     },
     {
       "epoch": 3.3670033670033668,
-      "grad_norm": 1.7532979249954224,
+      "grad_norm": 1.1592992544174194,
       "learning_rate": 3.628881406659185e-05,
-      "loss": 3.2499,
+      "loss": 3.2403,
       "step": 6000
     },
     {
       "epoch": 3.3670033670033668,
-      "eval_accuracy": 0.4066985195027942,
-      "eval_bleu": 0.1260881414008664,
-      "eval_loss": 3.3095922470092773,
-      "eval_perplexity": 27.373961381341072,
-      "eval_runtime": 12.162,
-      "eval_samples_per_second": 93.817,
-      "eval_steps_per_second": 1.48,
+      "eval_accuracy": 0.40832457337503847,
+      "eval_bleu": 0.129799477764246,
+      "eval_loss": 3.298476457595825,
+      "eval_perplexity": 27.07136311627636,
+      "eval_runtime": 12.1774,
+      "eval_samples_per_second": 93.699,
+      "eval_steps_per_second": 1.478,
       "step": 6000
     },
     {
       "epoch": 3.4231200897867566,
-      "grad_norm": 1.3179600238800049,
+      "grad_norm": 1.2478611469268799,
       "learning_rate": 3.504177578251652e-05,
-      "loss": 3.2306,
+      "loss": 3.2187,
       "step": 6100
     },
     {
       "epoch": 3.479236812570146,
-      "grad_norm": 3.5212039947509766,
+      "grad_norm": 1.8688626289367676,
       "learning_rate": 3.3794737498441205e-05,
-      "loss": 3.2421,
+      "loss": 3.2313,
       "step": 6200
     },
     {
       "epoch": 3.5353535353535355,
-      "grad_norm": 1.3254704475402832,
+      "grad_norm": 1.8166719675064087,
       "learning_rate": 3.254769921436588e-05,
-      "loss": 3.2305,
+      "loss": 3.2192,
       "step": 6300
     },
     {
       "epoch": 3.591470258136925,
-      "grad_norm": 3.1436195373535156,
+      "grad_norm": 1.6677237749099731,
       "learning_rate": 3.130066093029056e-05,
-      "loss": 3.2314,
+      "loss": 3.221,
       "step": 6400
     },
     {
       "epoch": 3.6475869809203143,
-      "grad_norm": 2.763392210006714,
+      "grad_norm": 1.4927235841751099,
       "learning_rate": 3.0053622646215242e-05,
-      "loss": 3.2277,
+      "loss": 3.2167,
       "step": 6500
     },
     {
       "epoch": 3.6475869809203143,
-      "eval_accuracy": 0.41002601857539517,
-      "eval_bleu": 0.12992587011026865,
-      "eval_loss": 3.2811858654022217,
-      "eval_perplexity": 26.607306683090435,
-      "eval_runtime": 12.2242,
-      "eval_samples_per_second": 93.34,
-      "eval_steps_per_second": 1.472,
+      "eval_accuracy": 0.4111534616185319,
+      "eval_bleu": 0.1288262427188711,
+      "eval_loss": 3.269272565841675,
+      "eval_perplexity": 26.292206536048766,
+      "eval_runtime": 12.1618,
+      "eval_samples_per_second": 93.819,
+      "eval_steps_per_second": 1.48,
       "step": 6500
     },
     {
       "epoch": 3.7037037037037037,
-      "grad_norm": 1.2504295110702515,
+      "grad_norm": 1.9368300437927246,
       "learning_rate": 2.880658436213992e-05,
-      "loss": 3.219,
+      "loss": 3.2084,
       "step": 6600
     },
     {
       "epoch": 3.759820426487093,
-      "grad_norm": 1.316945195198059,
+      "grad_norm": 1.1655679941177368,
       "learning_rate": 2.7559546078064598e-05,
-      "loss": 3.2109,
+      "loss": 3.1997,
       "step": 6700
     },
     {
       "epoch": 3.8159371492704826,
-      "grad_norm": 1.9210931062698364,
+      "grad_norm": 1.2166575193405151,
       "learning_rate": 2.6312507793989276e-05,
-      "loss": 3.1992,
+      "loss": 3.1889,
       "step": 6800
     },
     {
       "epoch": 3.872053872053872,
-      "grad_norm": 1.6544625759124756,
+      "grad_norm": 1.2495133876800537,
       "learning_rate": 2.5065469509913957e-05,
-      "loss": 3.1897,
+      "loss": 3.1814,
       "step": 6900
     },
     {
       "epoch": 3.9281705948372614,
-      "grad_norm": 1.2679674625396729,
+      "grad_norm": 1.4042552709579468,
       "learning_rate": 2.3818431225838632e-05,
-      "loss": 3.1992,
+      "loss": 3.1903,
       "step": 7000
     },
     {
       "epoch": 3.9281705948372614,
-      "eval_accuracy": 0.4128223514726582,
-      "eval_bleu": 0.1304892476483967,
-      "eval_loss": 3.252329111099243,
-      "eval_perplexity": 25.850478491671673,
-      "eval_runtime": 12.1672,
-      "eval_samples_per_second": 93.777,
+      "eval_accuracy": 0.41343918961176035,
+      "eval_bleu": 0.13053470151276936,
+      "eval_loss": 3.2455883026123047,
+      "eval_perplexity": 25.67681135292233,
+      "eval_runtime": 12.1742,
+      "eval_samples_per_second": 93.723,
       "eval_steps_per_second": 1.479,
       "step": 7000
     },
     {
       "epoch": 3.984287317620651,
-      "grad_norm": 1.1226177215576172,
+      "grad_norm": 1.472741723060608,
       "learning_rate": 2.2571392941763313e-05,
-      "loss": 3.1958,
+      "loss": 3.1863,
       "step": 7100
     },
     {
       "epoch": 4.040404040404041,
-      "grad_norm": 1.0558810234069824,
+      "grad_norm": 1.0730637311935425,
       "learning_rate": 2.132435465768799e-05,
-      "loss": 3.13,
+      "loss": 3.1197,
       "step": 7200
     },
     {
       "epoch": 4.09652076318743,
-      "grad_norm": 1.1988404989242554,
+      "grad_norm": 1.275009036064148,
       "learning_rate": 2.007731637361267e-05,
-      "loss": 3.1249,
+      "loss": 3.1142,
       "step": 7300
     },
     {
       "epoch": 4.1526374859708195,
-      "grad_norm": 1.914546251296997,
+      "grad_norm": 1.156083583831787,
       "learning_rate": 1.883027808953735e-05,
-      "loss": 3.1276,
+      "loss": 3.1174,
       "step": 7400
     },
     {
       "epoch": 4.2087542087542085,
-      "grad_norm": 1.7904256582260132,
+      "grad_norm": 1.0556427240371704,
       "learning_rate": 1.758323980546203e-05,
-      "loss": 3.13,
+      "loss": 3.1212,
       "step": 7500
     },
     {
       "epoch": 4.2087542087542085,
-      "eval_accuracy": 0.4154396299656541,
-      "eval_bleu": 0.13260343650795992,
-      "eval_loss": 3.233205795288086,
-      "eval_perplexity": 25.360828416165713,
-      "eval_runtime": 12.2696,
-      "eval_samples_per_second": 92.994,
-      "eval_steps_per_second": 1.467,
+      "eval_accuracy": 0.4161310027132311,
+      "eval_bleu": 0.1325477390562919,
+      "eval_loss": 3.2261738777160645,
+      "eval_perplexity": 25.1831187134555,
+      "eval_runtime": 12.1902,
+      "eval_samples_per_second": 93.6,
+      "eval_steps_per_second": 1.477,
       "step": 7500
     },
     {
       "epoch": 4.264870931537598,
-      "grad_norm": 2.1336634159088135,
+      "grad_norm": 2.7228903770446777,
       "learning_rate": 1.6336201521386706e-05,
-      "loss": 3.1166,
+      "loss": 3.1073,
       "step": 7600
     },
     {
       "epoch": 4.320987654320987,
-      "grad_norm": 1.4960366487503052,
+      "grad_norm": 1.526665210723877,
       "learning_rate": 1.5089163237311384e-05,
-      "loss": 3.1033,
+      "loss": 3.0936,
       "step": 7700
     },
     {
       "epoch": 4.377104377104377,
-      "grad_norm": 1.4756426811218262,
+      "grad_norm": 1.0413861274719238,
       "learning_rate": 1.3842124953236066e-05,
-      "loss": 3.1145,
+      "loss": 3.104,
       "step": 7800
     },
     {
       "epoch": 4.433221099887767,
-      "grad_norm": 1.741774082183838,
+      "grad_norm": 1.394884467124939,
       "learning_rate": 1.2595086669160744e-05,
-      "loss": 3.1115,
+      "loss": 3.1018,
       "step": 7900
     },
     {
       "epoch": 4.489337822671156,
-      "grad_norm": 1.0106322765350342,
+      "grad_norm": 1.2792820930480957,
       "learning_rate": 1.1348048385085423e-05,
-      "loss": 3.0915,
+      "loss": 3.0816,
       "step": 8000
     },
     {
       "epoch": 4.489337822671156,
-      "eval_accuracy": 0.4168352262553727,
-      "eval_bleu": 0.1317196572953092,
-      "eval_loss": 3.220038414001465,
-      "eval_perplexity": 25.029081630049546,
-      "eval_runtime": 12.1779,
-      "eval_samples_per_second": 93.694,
-      "eval_steps_per_second": 1.478,
+      "eval_accuracy": 0.4176114142470762,
+      "eval_bleu": 0.13073000122523024,
+      "eval_loss": 3.2127764225006104,
+      "eval_perplexity": 24.847979030639497,
+      "eval_runtime": 12.2049,
+      "eval_samples_per_second": 93.487,
+      "eval_steps_per_second": 1.475,
       "step": 8000
     },
     {
       "epoch": 4.545454545454545,
-      "grad_norm": 1.2306262254714966,
+      "grad_norm": 1.173828363418579,
       "learning_rate": 1.0101010101010101e-05,
-      "loss": 3.1086,
+      "loss": 3.0993,
       "step": 8100
     },
     {
       "epoch": 4.601571268237935,
-      "grad_norm": 1.141204833984375,
+      "grad_norm": 1.09369957447052,
       "learning_rate": 8.853971816934781e-06,
-      "loss": 3.0995,
+      "loss": 3.0905,
       "step": 8200
     },
     {
       "epoch": 4.657687991021325,
-      "grad_norm": 2.441620349884033,
+      "grad_norm": 1.0657522678375244,
       "learning_rate": 7.606933532859459e-06,
-      "loss": 3.106,
+      "loss": 3.097,
       "step": 8300
     },
     {
       "epoch": 4.713804713804714,
-      "grad_norm": 0.9955108761787415,
+      "grad_norm": 1.1146267652511597,
       "learning_rate": 6.359895248784138e-06,
-      "loss": 3.0982,
+      "loss": 3.0896,
       "step": 8400
     },
     {
       "epoch": 4.7699214365881035,
-      "grad_norm": 1.119258165359497,
+      "grad_norm": 2.1886322498321533,
       "learning_rate": 5.112856964708817e-06,
-      "loss": 3.1011,
+      "loss": 3.0917,
       "step": 8500
     },
     {
       "epoch": 4.7699214365881035,
-      "eval_accuracy": 0.4185786507179739,
-      "eval_bleu": 0.13349497596917598,
-      "eval_loss": 3.2064151763916016,
-      "eval_perplexity": 24.690416598736768,
-      "eval_runtime": 12.2406,
-      "eval_samples_per_second": 93.214,
-      "eval_steps_per_second": 1.471,
+      "eval_accuracy": 0.4195613081423491,
+      "eval_bleu": 0.13385511134755296,
+      "eval_loss": 3.1984846591949463,
+      "eval_perplexity": 24.49538320533492,
+      "eval_runtime": 12.1678,
+      "eval_samples_per_second": 93.772,
+      "eval_steps_per_second": 1.479,
       "step": 8500
     },
     {
       "epoch": 4.8260381593714925,
-      "grad_norm": 1.438043236732483,
+      "grad_norm": 1.1058974266052246,
       "learning_rate": 3.865818680633495e-06,
-      "loss": 3.0915,
+      "loss": 3.0822,
       "step": 8600
     },
     {
       "epoch": 4.882154882154882,
-      "grad_norm": 1.4048082828521729,
+      "grad_norm": 1.3942530155181885,
       "learning_rate": 2.6187803965581742e-06,
-      "loss": 3.1015,
+      "loss": 3.0923,
       "step": 8700
     },
     {
       "epoch": 4.938271604938271,
-      "grad_norm": 1.0656932592391968,
+      "grad_norm": 1.3995310068130493,
       "learning_rate": 1.3717421124828533e-06,
-      "loss": 3.0977,
+      "loss": 3.088,
       "step": 8800
     },
     {
       "epoch": 4.994388327721661,
-      "grad_norm": 0.9609319567680359,
+      "grad_norm": 1.1968387365341187,
       "learning_rate": 1.2470382840753213e-07,
-      "loss": 3.0882,
+      "loss": 3.0789,
       "step": 8900
     },
     {
       "epoch": 5.0,
       "step": 8910,
       "total_flos": 1.0587061010143642e+18,
-      "train_loss": 3.7750147520893753,
-      "train_runtime": 15697.9119,
-      "train_samples_per_second": 36.309,
-      "train_steps_per_second": 0.568
+      "train_loss": 3.767289323945907,
+      "train_runtime": 15738.6672,
+      "train_samples_per_second": 36.215,
+      "train_steps_per_second": 0.566
     }
   ],
   "logging_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9698132b7711dafc8a37071c574a7e35f5549f06a7d2a7c7a9c45ef5912876ca
+oid sha256:a1531590f5a306e0256e546c80e2541d64ed78f3ae0db27f5072411a0a6af9e8
 size 5560