Training in progress, step 3420

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:239d42f619e9c7ef78a589e8878755b7ff05452678bd76c3387d95b909c97859
+oid sha256:dc6b220a7ddc1693f8f9890cc5852a7b7e0f32a9fe123095302817600f977b31
 size 990185320

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc13493b7d44375cc8cae4d02fa249621dda652ef7ed46c67218c1d0e8ed9508
+oid sha256:5b3dcd6af690fe95d99438cd0881674cf89b2c2354b97a0b8dbee8da92dc4dfe
 size 1980545291

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46665dae441c58ae29353356f7aac4a3e2cfc255bb6a3218b134f74d51910343
+oid sha256:640f46eb39b244e7bfc751c4f25ad8e0191fac059c44f640e0680bc644444835
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d718bae36912f89382302637b1931f50f428ec0f6caf053ec44b1bbc42dc924d
+oid sha256:c693fb2ea94de3ab99065331d59deff37c4c392b1695d7781427a4e71d863f58
 size 1465

last-checkpoint/trainer_state.json

@@ -2,320 +2,396 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.871345029239766,
+  "epoch": 1.1918951132300357,
   "eval_steps": 400,
-  "global_step": 3200,
+  "global_step": 4000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.05847953216374269,
-      "grad_norm": 3.643113851547241,
-      "learning_rate": 4.3697368421052636e-05,
-      "loss": 1.2757,
+      "epoch": 0.029797377830750895,
+      "grad_norm": 4.59083890914917,
+      "learning_rate": 3.448376042908224e-05,
+      "loss": 1.4613,
       "step": 100
     },
     {
-      "epoch": 0.11695906432748537,
-      "grad_norm": 3.530352830886841,
-      "learning_rate": 4.2381578947368426e-05,
-      "loss": 1.337,
+      "epoch": 0.05959475566150179,
+      "grad_norm": 6.048128604888916,
+      "learning_rate": 3.39623063170441e-05,
+      "loss": 1.3655,
       "step": 200
     },
     {
-      "epoch": 0.17543859649122806,
-      "grad_norm": 5.174887657165527,
-      "learning_rate": 4.1065789473684215e-05,
-      "loss": 1.3173,
+      "epoch": 0.08939213349225268,
+      "grad_norm": 5.665088176727295,
+      "learning_rate": 3.344085220500596e-05,
+      "loss": 1.3917,
       "step": 300
     },
     {
-      "epoch": 0.23391812865497075,
-      "grad_norm": 3.0970983505249023,
-      "learning_rate": 3.9750000000000004e-05,
-      "loss": 1.3895,
+      "epoch": 0.11918951132300358,
+      "grad_norm": 5.817560195922852,
+      "learning_rate": 3.291939809296781e-05,
+      "loss": 1.3542,
       "step": 400
     },
     {
-      "epoch": 0.23391812865497075,
-      "eval_bleu": 39.47529188348838,
-      "eval_chrf": 65.3378689248108,
-      "eval_loss": 1.3580824136734009,
-      "eval_runtime": 56.39,
-      "eval_samples_per_second": 9.115,
-      "eval_steps_per_second": 1.153,
+      "epoch": 0.11918951132300358,
+      "eval_bleu": 37.614257412118484,
+      "eval_chrf": 64.19881893879953,
+      "eval_loss": 1.4418301582336426,
+      "eval_runtime": 59.317,
+      "eval_samples_per_second": 8.665,
+      "eval_steps_per_second": 1.096,
       "step": 400
     },
     {
-      "epoch": 0.29239766081871343,
-      "grad_norm": 5.711610794067383,
-      "learning_rate": 3.843421052631579e-05,
-      "loss": 1.3278,
+      "epoch": 0.14898688915375446,
+      "grad_norm": 5.501027584075928,
+      "learning_rate": 3.2397943980929674e-05,
+      "loss": 1.2908,
       "step": 500
     },
     {
-      "epoch": 0.3508771929824561,
-      "grad_norm": 3.5587220191955566,
-      "learning_rate": 3.711842105263158e-05,
-      "loss": 1.412,
+      "epoch": 0.17878426698450536,
+      "grad_norm": 7.59045934677124,
+      "learning_rate": 3.1876489868891536e-05,
+      "loss": 1.2857,
       "step": 600
     },
     {
-      "epoch": 0.4093567251461988,
-      "grad_norm": 3.550055980682373,
-      "learning_rate": 3.580263157894737e-05,
-      "loss": 1.3132,
+      "epoch": 0.20858164481525626,
+      "grad_norm": 5.808661937713623,
+      "learning_rate": 3.13550357568534e-05,
+      "loss": 1.3866,
       "step": 700
     },
     {
-      "epoch": 0.4678362573099415,
-      "grad_norm": 3.751816511154175,
-      "learning_rate": 3.448684210526316e-05,
-      "loss": 1.3409,
+      "epoch": 0.23837902264600716,
+      "grad_norm": 4.547348499298096,
+      "learning_rate": 3.083358164481525e-05,
+      "loss": 1.3608,
       "step": 800
     },
     {
-      "epoch": 0.4678362573099415,
-      "eval_bleu": 39.42249073542359,
-      "eval_chrf": 65.4339382658089,
-      "eval_loss": 1.34471595287323,
-      "eval_runtime": 56.6324,
-      "eval_samples_per_second": 9.076,
-      "eval_steps_per_second": 1.148,
+      "epoch": 0.23837902264600716,
+      "eval_bleu": 37.30665125911135,
+      "eval_chrf": 64.10679124808756,
+      "eval_loss": 1.4419689178466797,
+      "eval_runtime": 60.8532,
+      "eval_samples_per_second": 8.447,
+      "eval_steps_per_second": 1.068,
       "step": 800
     },
     {
-      "epoch": 0.5263157894736842,
-      "grad_norm": 3.660346508026123,
-      "learning_rate": 3.317105263157895e-05,
-      "loss": 1.3487,
+      "epoch": 0.26817640047675806,
+      "grad_norm": 5.98895263671875,
+      "learning_rate": 3.031212753277711e-05,
+      "loss": 1.2325,
       "step": 900
     },
     {
-      "epoch": 0.5847953216374269,
-      "grad_norm": 6.453419208526611,
-      "learning_rate": 3.185526315789474e-05,
-      "loss": 1.3217,
+      "epoch": 0.29797377830750893,
+      "grad_norm": 3.8415517807006836,
+      "learning_rate": 2.9790673420738973e-05,
+      "loss": 1.3219,
       "step": 1000
     },
     {
-      "epoch": 0.6432748538011696,
-      "grad_norm": 4.79046630859375,
-      "learning_rate": 3.053947368421053e-05,
-      "loss": 1.3755,
+      "epoch": 0.32777115613825986,
+      "grad_norm": 4.784970283508301,
+      "learning_rate": 2.9269219308700832e-05,
+      "loss": 1.2864,
       "step": 1100
     },
     {
-      "epoch": 0.7017543859649122,
-      "grad_norm": 4.565652847290039,
-      "learning_rate": 2.9223684210526318e-05,
-      "loss": 1.3523,
+      "epoch": 0.3575685339690107,
+      "grad_norm": 5.327032089233398,
+      "learning_rate": 2.8747765196662694e-05,
+      "loss": 1.2509,
       "step": 1200
     },
     {
-      "epoch": 0.7017543859649122,
-      "eval_bleu": 40.520638506139925,
-      "eval_chrf": 65.95395212452995,
-      "eval_loss": 1.3250073194503784,
-      "eval_runtime": 56.4151,
-      "eval_samples_per_second": 9.111,
-      "eval_steps_per_second": 1.152,
+      "epoch": 0.3575685339690107,
+      "eval_bleu": 37.69409614318079,
+      "eval_chrf": 64.1652126034879,
+      "eval_loss": 1.4356836080551147,
+      "eval_runtime": 61.5641,
+      "eval_samples_per_second": 8.349,
+      "eval_steps_per_second": 1.056,
       "step": 1200
     },
     {
-      "epoch": 0.7602339181286549,
-      "grad_norm": 3.959063768386841,
-      "learning_rate": 2.790789473684211e-05,
-      "loss": 1.3696,
+      "epoch": 0.3873659117997616,
+      "grad_norm": 6.48747444152832,
+      "learning_rate": 2.822631108462455e-05,
+      "loss": 1.3008,
       "step": 1300
     },
     {
-      "epoch": 0.8187134502923976,
-      "grad_norm": 5.297095775604248,
-      "learning_rate": 2.6592105263157896e-05,
-      "loss": 1.33,
+      "epoch": 0.4171632896305125,
+      "grad_norm": 10.605354309082031,
+      "learning_rate": 2.7704856972586408e-05,
+      "loss": 1.1936,
       "step": 1400
     },
     {
-      "epoch": 0.8771929824561403,
-      "grad_norm": 4.624599933624268,
-      "learning_rate": 2.527631578947369e-05,
-      "loss": 1.3566,
+      "epoch": 0.4469606674612634,
+      "grad_norm": 5.0671796798706055,
+      "learning_rate": 2.718340286054827e-05,
+      "loss": 1.2895,
       "step": 1500
     },
     {
-      "epoch": 0.935672514619883,
-      "grad_norm": 4.531945705413818,
-      "learning_rate": 2.3960526315789475e-05,
-      "loss": 1.3527,
+      "epoch": 0.4767580452920143,
+      "grad_norm": 3.872220516204834,
+      "learning_rate": 2.6661948748510128e-05,
+      "loss": 1.3018,
       "step": 1600
     },
     {
-      "epoch": 0.935672514619883,
-      "eval_bleu": 40.56125811248909,
-      "eval_chrf": 65.81915372622016,
-      "eval_loss": 1.3156400918960571,
-      "eval_runtime": 56.9807,
-      "eval_samples_per_second": 9.021,
-      "eval_steps_per_second": 1.141,
+      "epoch": 0.4767580452920143,
+      "eval_bleu": 37.773400824938314,
+      "eval_chrf": 64.3403222198125,
+      "eval_loss": 1.4320310354232788,
+      "eval_runtime": 61.3981,
+      "eval_samples_per_second": 8.372,
+      "eval_steps_per_second": 1.059,
       "step": 1600
     },
     {
-      "epoch": 0.9941520467836257,
-      "grad_norm": 4.245266437530518,
-      "learning_rate": 2.2644736842105267e-05,
-      "loss": 1.3855,
+      "epoch": 0.5065554231227652,
+      "grad_norm": 5.484747409820557,
+      "learning_rate": 2.614049463647199e-05,
+      "loss": 1.2971,
       "step": 1700
     },
     {
-      "epoch": 1.0526315789473684,
-      "grad_norm": 5.7170867919921875,
-      "learning_rate": 2.1328947368421053e-05,
-      "loss": 1.3195,
+      "epoch": 0.5363528009535161,
+      "grad_norm": 5.198516368865967,
+      "learning_rate": 2.561904052443385e-05,
+      "loss": 1.2132,
       "step": 1800
     },
     {
-      "epoch": 1.1111111111111112,
-      "grad_norm": 4.01872444152832,
-      "learning_rate": 2.0013157894736842e-05,
-      "loss": 1.2638,
+      "epoch": 0.566150178784267,
+      "grad_norm": 4.3765363693237305,
+      "learning_rate": 2.5097586412395707e-05,
+      "loss": 1.3134,
       "step": 1900
     },
     {
-      "epoch": 1.1695906432748537,
-      "grad_norm": 5.8071441650390625,
-      "learning_rate": 1.869736842105263e-05,
-      "loss": 1.3409,
+      "epoch": 0.5959475566150179,
+      "grad_norm": 5.041851997375488,
+      "learning_rate": 2.4576132300357566e-05,
+      "loss": 1.2662,
       "step": 2000
     },
     {
-      "epoch": 1.1695906432748537,
-      "eval_bleu": 40.43566420212,
-      "eval_chrf": 66.0648038736721,
-      "eval_loss": 1.3172210454940796,
-      "eval_runtime": 57.0925,
-      "eval_samples_per_second": 9.003,
-      "eval_steps_per_second": 1.139,
+      "epoch": 0.5959475566150179,
+      "eval_bleu": 38.239729003886104,
+      "eval_chrf": 64.08872351391828,
+      "eval_loss": 1.4284172058105469,
+      "eval_runtime": 61.8577,
+      "eval_samples_per_second": 8.309,
+      "eval_steps_per_second": 1.051,
       "step": 2000
     },
     {
-      "epoch": 1.2280701754385965,
-      "grad_norm": 5.8771162033081055,
-      "learning_rate": 1.738157894736842e-05,
-      "loss": 1.2521,
+      "epoch": 0.6257449344457687,
+      "grad_norm": 5.912806987762451,
+      "learning_rate": 2.4054678188319424e-05,
+      "loss": 1.2808,
       "step": 2100
     },
     {
-      "epoch": 1.286549707602339,
-      "grad_norm": 3.5148508548736572,
-      "learning_rate": 1.606578947368421e-05,
-      "loss": 1.2719,
+      "epoch": 0.6555423122765197,
+      "grad_norm": 7.626971244812012,
+      "learning_rate": 2.3533224076281286e-05,
+      "loss": 1.265,
       "step": 2200
     },
     {
-      "epoch": 1.345029239766082,
-      "grad_norm": 3.7268385887145996,
-      "learning_rate": 1.4750000000000001e-05,
-      "loss": 1.3249,
+      "epoch": 0.6853396901072706,
+      "grad_norm": 6.593811511993408,
+      "learning_rate": 2.3011769964243145e-05,
+      "loss": 1.236,
       "step": 2300
     },
     {
-      "epoch": 1.4035087719298245,
-      "grad_norm": 6.356854438781738,
-      "learning_rate": 1.343421052631579e-05,
-      "loss": 1.2397,
+      "epoch": 0.7151370679380215,
+      "grad_norm": 5.527437686920166,
+      "learning_rate": 2.2490315852205003e-05,
+      "loss": 1.2324,
       "step": 2400
     },
     {
-      "epoch": 1.4035087719298245,
-      "eval_bleu": 40.582868133203924,
-      "eval_chrf": 65.94447550029828,
-      "eval_loss": 1.321014642715454,
-      "eval_runtime": 56.2984,
-      "eval_samples_per_second": 9.13,
-      "eval_steps_per_second": 1.155,
+      "epoch": 0.7151370679380215,
+      "eval_bleu": 38.382663636027985,
+      "eval_chrf": 64.41707679357026,
+      "eval_loss": 1.4260649681091309,
+      "eval_runtime": 61.867,
+      "eval_samples_per_second": 8.308,
+      "eval_steps_per_second": 1.051,
       "step": 2400
     },
     {
-      "epoch": 1.4619883040935673,
-      "grad_norm": 4.95539665222168,
-      "learning_rate": 1.211842105263158e-05,
-      "loss": 1.3243,
+      "epoch": 0.7449344457687723,
+      "grad_norm": 6.579026222229004,
+      "learning_rate": 2.1968861740166865e-05,
+      "loss": 1.2627,
       "step": 2500
     },
     {
-      "epoch": 1.52046783625731,
-      "grad_norm": 3.4667985439300537,
-      "learning_rate": 1.0802631578947369e-05,
-      "loss": 1.2947,
+      "epoch": 0.7747318235995232,
+      "grad_norm": 5.275082111358643,
+      "learning_rate": 2.1447407628128724e-05,
+      "loss": 1.2462,
       "step": 2600
     },
     {
-      "epoch": 1.5789473684210527,
-      "grad_norm": 3.2790651321411133,
-      "learning_rate": 9.486842105263158e-06,
-      "loss": 1.2674,
+      "epoch": 0.8045292014302742,
+      "grad_norm": 8.691877365112305,
+      "learning_rate": 2.0925953516090586e-05,
+      "loss": 1.1717,
       "step": 2700
     },
     {
-      "epoch": 1.6374269005847952,
-      "grad_norm": 4.367522716522217,
-      "learning_rate": 8.171052631578949e-06,
-      "loss": 1.3275,
+      "epoch": 0.834326579261025,
+      "grad_norm": 6.7765913009643555,
+      "learning_rate": 2.040449940405244e-05,
+      "loss": 1.2225,
       "step": 2800
     },
     {
-      "epoch": 1.6374269005847952,
-      "eval_bleu": 40.196557878597055,
-      "eval_chrf": 66.05405159197521,
-      "eval_loss": 1.3174731731414795,
-      "eval_runtime": 55.9756,
-      "eval_samples_per_second": 9.183,
-      "eval_steps_per_second": 1.161,
+      "epoch": 0.834326579261025,
+      "eval_bleu": 38.71626609686758,
+      "eval_chrf": 64.87666686265256,
+      "eval_loss": 1.4296128749847412,
+      "eval_runtime": 60.8774,
+      "eval_samples_per_second": 8.443,
+      "eval_steps_per_second": 1.068,
       "step": 2800
     },
     {
-      "epoch": 1.695906432748538,
-      "grad_norm": 3.6613388061523438,
-      "learning_rate": 6.855263157894738e-06,
-      "loss": 1.2249,
+      "epoch": 0.8641239570917759,
+      "grad_norm": 7.38505744934082,
+      "learning_rate": 1.98830452920143e-05,
+      "loss": 1.2582,
       "step": 2900
     },
     {
-      "epoch": 1.7543859649122808,
-      "grad_norm": 4.5111002922058105,
-      "learning_rate": 5.5394736842105266e-06,
-      "loss": 1.2362,
+      "epoch": 0.8939213349225268,
+      "grad_norm": 5.248499870300293,
+      "learning_rate": 1.936159117997616e-05,
+      "loss": 1.2588,
       "step": 3000
     },
     {
-      "epoch": 1.8128654970760234,
-      "grad_norm": 5.002144813537598,
-      "learning_rate": 4.223684210526316e-06,
-      "loss": 1.2918,
+      "epoch": 0.9237187127532777,
+      "grad_norm": 4.9279069900512695,
+      "learning_rate": 1.884013706793802e-05,
+      "loss": 1.2679,
       "step": 3100
     },
     {
-      "epoch": 1.871345029239766,
-      "grad_norm": 3.4458835124969482,
-      "learning_rate": 2.9078947368421054e-06,
-      "loss": 1.2348,
+      "epoch": 0.9535160905840286,
+      "grad_norm": 7.552145004272461,
+      "learning_rate": 1.831868295589988e-05,
+      "loss": 1.2384,
       "step": 3200
     },
     {
-      "epoch": 1.871345029239766,
-      "eval_bleu": 39.817105604910104,
-      "eval_chrf": 65.83702875844537,
-      "eval_loss": 1.3151392936706543,
-      "eval_runtime": 56.5728,
-      "eval_samples_per_second": 9.086,
-      "eval_steps_per_second": 1.149,
+      "epoch": 0.9535160905840286,
+      "eval_bleu": 38.518218505947324,
+      "eval_chrf": 64.89625356432627,
+      "eval_loss": 1.417944312095642,
+      "eval_runtime": 61.2351,
+      "eval_samples_per_second": 8.394,
+      "eval_steps_per_second": 1.061,
       "step": 3200
+    },
+    {
+      "epoch": 0.9833134684147795,
+      "grad_norm": 5.52092170715332,
+      "learning_rate": 1.779722884386174e-05,
+      "loss": 1.2863,
+      "step": 3300
+    },
+    {
+      "epoch": 1.0131108462455305,
+      "grad_norm": 5.495266437530518,
+      "learning_rate": 1.72757747318236e-05,
+      "loss": 1.1856,
+      "step": 3400
+    },
+    {
+      "epoch": 1.0429082240762813,
+      "grad_norm": 7.5805511474609375,
+      "learning_rate": 1.6754320619785457e-05,
+      "loss": 1.147,
+      "step": 3500
+    },
+    {
+      "epoch": 1.0727056019070322,
+      "grad_norm": 5.156323432922363,
+      "learning_rate": 1.6232866507747316e-05,
+      "loss": 1.1485,
+      "step": 3600
+    },
+    {
+      "epoch": 1.0727056019070322,
+      "eval_bleu": 38.668867292632044,
+      "eval_chrf": 64.79576644290259,
+      "eval_loss": 1.4277405738830566,
+      "eval_runtime": 60.538,
+      "eval_samples_per_second": 8.491,
+      "eval_steps_per_second": 1.074,
+      "step": 3600
+    },
+    {
+      "epoch": 1.102502979737783,
+      "grad_norm": 4.9568867683410645,
+      "learning_rate": 1.5711412395709178e-05,
+      "loss": 1.157,
+      "step": 3700
+    },
+    {
+      "epoch": 1.132300357568534,
+      "grad_norm": 4.534054279327393,
+      "learning_rate": 1.5189958283671036e-05,
+      "loss": 1.2088,
+      "step": 3800
+    },
+    {
+      "epoch": 1.162097735399285,
+      "grad_norm": 5.1630635261535645,
+      "learning_rate": 1.4668504171632896e-05,
+      "loss": 1.1492,
+      "step": 3900
+    },
+    {
+      "epoch": 1.1918951132300357,
+      "grad_norm": 6.447033405303955,
+      "learning_rate": 1.4147050059594753e-05,
+      "loss": 1.1213,
+      "step": 4000
+    },
+    {
+      "epoch": 1.1918951132300357,
+      "eval_bleu": 39.1303880259221,
+      "eval_chrf": 64.98118735600528,
+      "eval_loss": 1.4224255084991455,
+      "eval_runtime": 59.6926,
+      "eval_samples_per_second": 8.611,
+      "eval_steps_per_second": 1.089,
+      "step": 4000
     }
   ],
   "logging_steps": 100,
-  "max_steps": 3420,
+  "max_steps": 6712,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 400,
@@ -331,7 +407,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8764108937625600.0,
+  "total_flos": 1.0955136172032e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62b6e79a3c177bbec778e900c4bdda3fb42784f15e16b06128f5ef92437fb78c
+oid sha256:1296a4c4cbbfb511129c5fc25155a089f73ccf8250b3689b0cf311c5f320fd65
 size 5905

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:239d42f619e9c7ef78a589e8878755b7ff05452678bd76c3387d95b909c97859
+oid sha256:87b5562ff57c9718f87f2bf333235b1a1bd61d2e3cd5b19f6faf8e9b411d4c66
 size 990185320