Training in progress, step 1078, checkpoint

last-checkpoint/optimizer_0/__0_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a352f0d84009b1817ea378a4704c01130220431cda057a719176edb53b9ce38
+oid sha256:c1423094be38f2b5ae113f9990f9d961754f62cae91e49b685bfcf23e02eb7bb
 size 13934748

last-checkpoint/optimizer_0/__1_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:192c82f34e86d685c6f351fd58c1000ddea9a13d640195ac79c49fbf42423aa5
+oid sha256:08f90b95cf50f53920f37b950eff873dabfa8ac64d13dc56f42397849a07fc07
 size 13999412

last-checkpoint/optimizer_0/__2_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b90ca6c9a0d45f633e326ad429b79dcb8a229254c394c0026c58947de8b6ccb1
+oid sha256:cd93a0a0d0644200d9fcbccac4f34181e77bde63b6d95229e509a8a350855523
 size 13990904

last-checkpoint/optimizer_0/__3_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1ddcd3b678ecc28638f1325c2c32db98cad1876b80914907eec102e20d65888
+oid sha256:e8ecc9cc52a997dd5614bbdc7880c6e20c1e932861d4d32c5e916f3e3d25f364
 size 13990904

last-checkpoint/pytorch_model_fsdp_0/__0_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:441eb9b06b4fc0f3fa0a9291de25b8426d0d9f412df64f69773da2db1b4860b2
+oid sha256:e17140212e09dfb8383f39aada365a791d0a38991ff14289444e40e32fbaa42f
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__1_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01659b87d6d23358ab75fa4077af9feedf08b369b1c157aa83e98851b9c0d1ee
+oid sha256:558da04d338fb499d79f7280741df812318a3c5c4aee5a107c7093df843fc4c8
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__2_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71450373e32f8a9a1b7bd7c09bbf7665cd2aab9935d9141b9e0d70c0fce7c3de
+oid sha256:4916f4007500b9580d9f5d30067dea1660a0a0d57759318fb72d6b96562ab22b
 size 6966784

last-checkpoint/pytorch_model_fsdp_0/__3_0.distcp

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:173eff09d590e65fe2dd1179e23f7fb059beaf649179bf2d537bde02e80545b0
+oid sha256:0548992fa9f7a727bc6d76af19be130091084cba363ceab858c1d0bb1f9cc034
 size 6966784

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6afe62f64f980792c5f93908f1252e0efd7d9d6dd9a401096016c0cf0f6e9df7
+oid sha256:13e6011477fddf03495b9f86e6d1296fafc3690e147017eae3cf4b4b45062459
 size 14960

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c96a88229c7cf8988c09092a9afef0bd222230400623a17d132e957aa024720
+oid sha256:b8280d014b0e56eff05e43f5a8c43d0264e1d5386ee80569ca7b06fed8e39dd5
 size 14960

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50d2280d0785bc9b8dd3a1397de7a4d5f6e608d8e08010244249962de0f0c423
+oid sha256:fc32ead699b38a508a624a41fb823b6da5d825da75c4d29d42299e951670eefa
 size 14960

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05e89f8a1132e0f0def133732be826c04d18fb1ddc8e499809e4f481802df182
+oid sha256:4ff9068623665c0763ac309d5165fb01837d68f9e74aa1848d43a5612394a0e9
 size 14960

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ead26a1aba46fa0b3384e323e0349ee0e9c3d6b20dad4ce8e9c9bf15675155cc
+oid sha256:706495f69aeec71fe1992742f4e3f2241879821efb572c1f0819a3bb4ed3108f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9276437847866419,
+  "epoch": 1.0,
   "eval_steps": 20,
-  "global_step": 1000,
+  "global_step": 1078,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -7619,6 +7619,588 @@
       "eval_samples_per_second": 5.766,
       "eval_steps_per_second": 0.188,
       "step": 1000
+    },
+    {
+      "epoch": 0.9285714285714286,
+      "grad_norm": 6.499805927276611,
+      "learning_rate": 3.093540055676958e-07,
+      "loss": 0.2015,
+      "step": 1001
+    },
+    {
+      "epoch": 0.9294990723562152,
+      "grad_norm": 4.742324352264404,
+      "learning_rate": 3.0141136285129825e-07,
+      "loss": 0.1585,
+      "step": 1002
+    },
+    {
+      "epoch": 0.9304267161410018,
+      "grad_norm": 4.395940780639648,
+      "learning_rate": 2.935704537404083e-07,
+      "loss": 0.1249,
+      "step": 1003
+    },
+    {
+      "epoch": 0.9313543599257885,
+      "grad_norm": 3.036573886871338,
+      "learning_rate": 2.8583136048245697e-07,
+      "loss": 0.1331,
+      "step": 1004
+    },
+    {
+      "epoch": 0.9322820037105751,
+      "grad_norm": 4.401485919952393,
+      "learning_rate": 2.781941642568686e-07,
+      "loss": 0.2138,
+      "step": 1005
+    },
+    {
+      "epoch": 0.9332096474953617,
+      "grad_norm": 4.973133087158203,
+      "learning_rate": 2.706589451742181e-07,
+      "loss": 0.2253,
+      "step": 1006
+    },
+    {
+      "epoch": 0.9341372912801484,
+      "grad_norm": 6.711733818054199,
+      "learning_rate": 2.632257822753881e-07,
+      "loss": 0.2465,
+      "step": 1007
+    },
+    {
+      "epoch": 0.935064935064935,
+      "grad_norm": 3.2245848178863525,
+      "learning_rate": 2.5589475353073987e-07,
+      "loss": 0.1524,
+      "step": 1008
+    },
+    {
+      "epoch": 0.9359925788497218,
+      "grad_norm": 3.8495306968688965,
+      "learning_rate": 2.486659358392951e-07,
+      "loss": 0.1646,
+      "step": 1009
+    },
+    {
+      "epoch": 0.9369202226345084,
+      "grad_norm": 5.713381290435791,
+      "learning_rate": 2.4153940502793185e-07,
+      "loss": 0.2161,
+      "step": 1010
+    },
+    {
+      "epoch": 0.937847866419295,
+      "grad_norm": 3.7789957523345947,
+      "learning_rate": 2.3451523585058756e-07,
+      "loss": 0.1509,
+      "step": 1011
+    },
+    {
+      "epoch": 0.9387755102040817,
+      "grad_norm": 4.0073652267456055,
+      "learning_rate": 2.2759350198746978e-07,
+      "loss": 0.1402,
+      "step": 1012
+    },
+    {
+      "epoch": 0.9397031539888683,
+      "grad_norm": 3.3210415840148926,
+      "learning_rate": 2.2077427604429435e-07,
+      "loss": 0.1431,
+      "step": 1013
+    },
+    {
+      "epoch": 0.9406307977736549,
+      "grad_norm": 5.2902750968933105,
+      "learning_rate": 2.1405762955151178e-07,
+      "loss": 0.1239,
+      "step": 1014
+    },
+    {
+      "epoch": 0.9415584415584416,
+      "grad_norm": 5.931840419769287,
+      "learning_rate": 2.0744363296356872e-07,
+      "loss": 0.1965,
+      "step": 1015
+    },
+    {
+      "epoch": 0.9424860853432282,
+      "grad_norm": 3.9065020084381104,
+      "learning_rate": 2.009323556581566e-07,
+      "loss": 0.1344,
+      "step": 1016
+    },
+    {
+      "epoch": 0.9434137291280148,
+      "grad_norm": 3.194225311279297,
+      "learning_rate": 1.9452386593549534e-07,
+      "loss": 0.0979,
+      "step": 1017
+    },
+    {
+      "epoch": 0.9443413729128015,
+      "grad_norm": 3.794304847717285,
+      "learning_rate": 1.8821823101760949e-07,
+      "loss": 0.2038,
+      "step": 1018
+    },
+    {
+      "epoch": 0.9452690166975881,
+      "grad_norm": 3.638219118118286,
+      "learning_rate": 1.8201551704762453e-07,
+      "loss": 0.1254,
+      "step": 1019
+    },
+    {
+      "epoch": 0.9461966604823747,
+      "grad_norm": 4.820856094360352,
+      "learning_rate": 1.7591578908907724e-07,
+      "loss": 0.1909,
+      "step": 1020
+    },
+    {
+      "epoch": 0.9461966604823747,
+      "eval_accuracy": 0.8580931263858093,
+      "eval_f1": 0.6966824644549763,
+      "eval_loss": 0.3055438697338104,
+      "eval_precision": 0.8698224852071006,
+      "eval_recall": 0.5810276679841897,
+      "eval_runtime": 47.7431,
+      "eval_samples_per_second": 5.781,
+      "eval_steps_per_second": 0.189,
+      "step": 1020
+    },
+    {
+      "epoch": 0.9471243042671614,
+      "grad_norm": 3.5032293796539307,
+      "learning_rate": 1.699191111252241e-07,
+      "loss": 0.077,
+      "step": 1021
+    },
+    {
+      "epoch": 0.948051948051948,
+      "grad_norm": 3.7761011123657227,
+      "learning_rate": 1.6402554605838173e-07,
+      "loss": 0.1564,
+      "step": 1022
+    },
+    {
+      "epoch": 0.9489795918367347,
+      "grad_norm": 5.951882839202881,
+      "learning_rate": 1.5823515570925763e-07,
+      "loss": 0.2323,
+      "step": 1023
+    },
+    {
+      "epoch": 0.9499072356215214,
+      "grad_norm": 4.935650825500488,
+      "learning_rate": 1.5254800081630828e-07,
+      "loss": 0.2172,
+      "step": 1024
+    },
+    {
+      "epoch": 0.950834879406308,
+      "grad_norm": 5.279281139373779,
+      "learning_rate": 1.469641410350964e-07,
+      "loss": 0.1845,
+      "step": 1025
+    },
+    {
+      "epoch": 0.9517625231910947,
+      "grad_norm": 5.034005641937256,
+      "learning_rate": 1.4148363493766803e-07,
+      "loss": 0.174,
+      "step": 1026
+    },
+    {
+      "epoch": 0.9526901669758813,
+      "grad_norm": 3.964360237121582,
+      "learning_rate": 1.361065400119399e-07,
+      "loss": 0.0888,
+      "step": 1027
+    },
+    {
+      "epoch": 0.9536178107606679,
+      "grad_norm": 4.862616062164307,
+      "learning_rate": 1.30832912661093e-07,
+      "loss": 0.2007,
+      "step": 1028
+    },
+    {
+      "epoch": 0.9545454545454546,
+      "grad_norm": 4.140252590179443,
+      "learning_rate": 1.2566280820298427e-07,
+      "loss": 0.13,
+      "step": 1029
+    },
+    {
+      "epoch": 0.9554730983302412,
+      "grad_norm": 5.299205780029297,
+      "learning_rate": 1.2059628086956044e-07,
+      "loss": 0.1795,
+      "step": 1030
+    },
+    {
+      "epoch": 0.9564007421150278,
+      "grad_norm": 5.694372653961182,
+      "learning_rate": 1.1563338380629618e-07,
+      "loss": 0.2278,
+      "step": 1031
+    },
+    {
+      "epoch": 0.9573283858998145,
+      "grad_norm": 4.10621452331543,
+      "learning_rate": 1.1077416907163573e-07,
+      "loss": 0.1832,
+      "step": 1032
+    },
+    {
+      "epoch": 0.9582560296846011,
+      "grad_norm": 3.0105836391448975,
+      "learning_rate": 1.0601868763643997e-07,
+      "loss": 0.1126,
+      "step": 1033
+    },
+    {
+      "epoch": 0.9591836734693877,
+      "grad_norm": 5.412391185760498,
+      "learning_rate": 1.0136698938346012e-07,
+      "loss": 0.1996,
+      "step": 1034
+    },
+    {
+      "epoch": 0.9601113172541744,
+      "grad_norm": 3.507596492767334,
+      "learning_rate": 9.68191231068083e-08,
+      "loss": 0.1647,
+      "step": 1035
+    },
+    {
+      "epoch": 0.961038961038961,
+      "grad_norm": 4.733442783355713,
+      "learning_rate": 9.237513651145224e-08,
+      "loss": 0.102,
+      "step": 1036
+    },
+    {
+      "epoch": 0.9619666048237476,
+      "grad_norm": 6.855641841888428,
+      "learning_rate": 8.80350762127058e-08,
+      "loss": 0.1197,
+      "step": 1037
+    },
+    {
+      "epoch": 0.9628942486085343,
+      "grad_norm": 3.809262275695801,
+      "learning_rate": 8.379898773574924e-08,
+      "loss": 0.1287,
+      "step": 1038
+    },
+    {
+      "epoch": 0.963821892393321,
+      "grad_norm": 3.4764761924743652,
+      "learning_rate": 7.966691551514527e-08,
+      "loss": 0.1106,
+      "step": 1039
+    },
+    {
+      "epoch": 0.9647495361781077,
+      "grad_norm": 5.395627021789551,
+      "learning_rate": 7.563890289437825e-08,
+      "loss": 0.2017,
+      "step": 1040
+    },
+    {
+      "epoch": 0.9647495361781077,
+      "eval_accuracy": 0.8580931263858093,
+      "eval_f1": 0.6952380952380952,
+      "eval_loss": 0.30581432580947876,
+      "eval_precision": 0.874251497005988,
+      "eval_recall": 0.5770750988142292,
+      "eval_runtime": 48.746,
+      "eval_samples_per_second": 5.662,
+      "eval_steps_per_second": 0.185,
+      "step": 1040
+    },
+    {
+      "epoch": 0.9656771799628943,
+      "grad_norm": 3.9932026863098145,
+      "learning_rate": 7.171499212539124e-08,
+      "loss": 0.1513,
+      "step": 1041
+    },
+    {
+      "epoch": 0.9666048237476809,
+      "grad_norm": 5.8107075691223145,
+      "learning_rate": 6.78952243681541e-08,
+      "loss": 0.172,
+      "step": 1042
+    },
+    {
+      "epoch": 0.9675324675324676,
+      "grad_norm": 3.7357017993927,
+      "learning_rate": 6.417963969022389e-08,
+      "loss": 0.1422,
+      "step": 1043
+    },
+    {
+      "epoch": 0.9684601113172542,
+      "grad_norm": 3.854876756668091,
+      "learning_rate": 6.056827706632185e-08,
+      "loss": 0.1587,
+      "step": 1044
+    },
+    {
+      "epoch": 0.9693877551020408,
+      "grad_norm": 6.006348133087158,
+      "learning_rate": 5.7061174377937015e-08,
+      "loss": 0.2244,
+      "step": 1045
+    },
+    {
+      "epoch": 0.9703153988868275,
+      "grad_norm": 4.745636463165283,
+      "learning_rate": 5.365836841291439e-08,
+      "loss": 0.1803,
+      "step": 1046
+    },
+    {
+      "epoch": 0.9712430426716141,
+      "grad_norm": 3.8510711193084717,
+      "learning_rate": 5.035989486508075e-08,
+      "loss": 0.1635,
+      "step": 1047
+    },
+    {
+      "epoch": 0.9721706864564007,
+      "grad_norm": 5.504276752471924,
+      "learning_rate": 4.716578833386054e-08,
+      "loss": 0.1517,
+      "step": 1048
+    },
+    {
+      "epoch": 0.9730983302411874,
+      "grad_norm": 3.3400299549102783,
+      "learning_rate": 4.4076082323920576e-08,
+      "loss": 0.1494,
+      "step": 1049
+    },
+    {
+      "epoch": 0.974025974025974,
+      "grad_norm": 5.584471225738525,
+      "learning_rate": 4.109080924481479e-08,
+      "loss": 0.1781,
+      "step": 1050
+    },
+    {
+      "epoch": 0.9749536178107606,
+      "grad_norm": 4.575666904449463,
+      "learning_rate": 3.82100004106456e-08,
+      "loss": 0.1298,
+      "step": 1051
+    },
+    {
+      "epoch": 0.9758812615955473,
+      "grad_norm": 3.4456560611724854,
+      "learning_rate": 3.543368603973529e-08,
+      "loss": 0.1292,
+      "step": 1052
+    },
+    {
+      "epoch": 0.9768089053803339,
+      "grad_norm": 2.841853618621826,
+      "learning_rate": 3.2761895254306285e-08,
+      "loss": 0.076,
+      "step": 1053
+    },
+    {
+      "epoch": 0.9777365491651205,
+      "grad_norm": 4.662397384643555,
+      "learning_rate": 3.019465608018024e-08,
+      "loss": 0.2181,
+      "step": 1054
+    },
+    {
+      "epoch": 0.9786641929499073,
+      "grad_norm": 3.259526014328003,
+      "learning_rate": 2.773199544648164e-08,
+      "loss": 0.108,
+      "step": 1055
+    },
+    {
+      "epoch": 0.9795918367346939,
+      "grad_norm": 4.32330322265625,
+      "learning_rate": 2.537393918535358e-08,
+      "loss": 0.2068,
+      "step": 1056
+    },
+    {
+      "epoch": 0.9805194805194806,
+      "grad_norm": 4.7618536949157715,
+      "learning_rate": 2.312051203169352e-08,
+      "loss": 0.1936,
+      "step": 1057
+    },
+    {
+      "epoch": 0.9814471243042672,
+      "grad_norm": 4.779612064361572,
+      "learning_rate": 2.0971737622883515e-08,
+      "loss": 0.1007,
+      "step": 1058
+    },
+    {
+      "epoch": 0.9823747680890538,
+      "grad_norm": 4.346301078796387,
+      "learning_rate": 1.8927638498551502e-08,
+      "loss": 0.1594,
+      "step": 1059
+    },
+    {
+      "epoch": 0.9833024118738405,
+      "grad_norm": 4.017016410827637,
+      "learning_rate": 1.698823610032929e-08,
+      "loss": 0.1828,
+      "step": 1060
+    },
+    {
+      "epoch": 0.9833024118738405,
+      "eval_accuracy": 0.8603104212860311,
+      "eval_f1": 0.7028301886792453,
+      "eval_loss": 0.306577205657959,
+      "eval_precision": 0.8713450292397661,
+      "eval_recall": 0.5889328063241107,
+      "eval_runtime": 47.7001,
+      "eval_samples_per_second": 5.786,
+      "eval_steps_per_second": 0.189,
+      "step": 1060
+    },
+    {
+      "epoch": 0.9842300556586271,
+      "grad_norm": 5.6250505447387695,
+      "learning_rate": 1.5153550771630498e-08,
+      "loss": 0.17,
+      "step": 1061
+    },
+    {
+      "epoch": 0.9851576994434137,
+      "grad_norm": 5.426425933837891,
+      "learning_rate": 1.3423601757436289e-08,
+      "loss": 0.2122,
+      "step": 1062
+    },
+    {
+      "epoch": 0.9860853432282004,
+      "grad_norm": 4.71135950088501,
+      "learning_rate": 1.179840720409331e-08,
+      "loss": 0.1715,
+      "step": 1063
+    },
+    {
+      "epoch": 0.987012987012987,
+      "grad_norm": 4.350978851318359,
+      "learning_rate": 1.0277984159122734e-08,
+      "loss": 0.1704,
+      "step": 1064
+    },
+    {
+      "epoch": 0.9879406307977736,
+      "grad_norm": 5.211360931396484,
+      "learning_rate": 8.862348571043733e-09,
+      "loss": 0.166,
+      "step": 1065
+    },
+    {
+      "epoch": 0.9888682745825603,
+      "grad_norm": 4.015779495239258,
+      "learning_rate": 7.551515289203615e-09,
+      "loss": 0.1616,
+      "step": 1066
+    },
+    {
+      "epoch": 0.9897959183673469,
+      "grad_norm": 4.356948375701904,
+      "learning_rate": 6.345498063622391e-09,
+      "loss": 0.1961,
+      "step": 1067
+    },
+    {
+      "epoch": 0.9907235621521335,
+      "grad_norm": 6.508297920227051,
+      "learning_rate": 5.2443095448506674e-09,
+      "loss": 0.1935,
+      "step": 1068
+    },
+    {
+      "epoch": 0.9916512059369202,
+      "grad_norm": 4.595229625701904,
+      "learning_rate": 4.247961283835311e-09,
+      "loss": 0.2116,
+      "step": 1069
+    },
+    {
+      "epoch": 0.9925788497217068,
+      "grad_norm": 4.39501428604126,
+      "learning_rate": 3.3564637317984318e-09,
+      "loss": 0.1568,
+      "step": 1070
+    },
+    {
+      "epoch": 0.9935064935064936,
+      "grad_norm": 4.391909599304199,
+      "learning_rate": 2.5698262401263607e-09,
+      "loss": 0.1553,
+      "step": 1071
+    },
+    {
+      "epoch": 0.9944341372912802,
+      "grad_norm": 3.195699453353882,
+      "learning_rate": 1.888057060274173e-09,
+      "loss": 0.1469,
+      "step": 1072
+    },
+    {
+      "epoch": 0.9953617810760668,
+      "grad_norm": 5.479938507080078,
+      "learning_rate": 1.3111633436779792e-09,
+      "loss": 0.1662,
+      "step": 1073
+    },
+    {
+      "epoch": 0.9962894248608535,
+      "grad_norm": 4.181588172912598,
+      "learning_rate": 8.391511416816489e-10,
+      "loss": 0.1746,
+      "step": 1074
+    },
+    {
+      "epoch": 0.9972170686456401,
+      "grad_norm": 4.144800662994385,
+      "learning_rate": 4.720254054679796e-10,
+      "loss": 0.1624,
+      "step": 1075
+    },
+    {
+      "epoch": 0.9981447124304267,
+      "grad_norm": 3.857682228088379,
+      "learning_rate": 2.0978998601206558e-10,
+      "loss": 0.1293,
+      "step": 1076
+    },
+    {
+      "epoch": 0.9990723562152134,
+      "grad_norm": 5.734769344329834,
+      "learning_rate": 5.244763404133046e-11,
+      "loss": 0.1897,
+      "step": 1077
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 3.7049508094787598,
+      "learning_rate": 0.0,
+      "loss": 0.1706,
+      "step": 1078
     }
   ],
   "logging_steps": 1,
@@ -7633,12 +8215,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.198993040534405e+17,
+  "total_flos": 3.4499307937307034e+17,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null