Training in progress, step 812, checkpoint

checkpoint-812/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:910e14490fdfe72dc83b26b51893e989d270d119489bfe948844faa00f70f7fd
+oid sha256:02c19f31040d1ed38148f0e354e587cb1918ce232218ee4a20fda15565c8e328
 size 4426572920

checkpoint-812/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d94c7b1e6a0eff38ccfca412c57770faf83700a2768814cb64d452159101848f
+oid sha256:70638c86bde1cf4d9537b77de54946a1e7f830f38752d13c72d354a224630b8c
 size 6858880301

checkpoint-812/trainer_state.json

@@ -13,575 +13,575 @@
       "epoch": 0.0018478026546764805,
       "grad_norm": 4.9375,
       "learning_rate": 4.41717791411043e-06,
-      "loss": 1.7101457595825196,
+      "loss": 1.7101329803466796,
       "step": 10
     },
     {
       "epoch": 0.003695605309352961,
       "grad_norm": 1.1015625,
       "learning_rate": 9.325153374233129e-06,
-      "loss": 1.5276947021484375,
+      "loss": 1.5276897430419922,
       "step": 20
     },
     {
       "epoch": 0.0055434079640294415,
       "grad_norm": 0.8828125,
       "learning_rate": 1.423312883435583e-05,
-      "loss": 1.420623207092285,
+      "loss": 1.4206634521484376,
       "step": 30
     },
     {
       "epoch": 0.007391210618705922,
-      "grad_norm": 0.74609375,
+      "grad_norm": 0.7421875,
       "learning_rate": 1.914110429447853e-05,
-      "loss": 1.387537956237793,
+      "loss": 1.3875360488891602,
       "step": 40
     },
     {
       "epoch": 0.009239013273382403,
       "grad_norm": 0.765625,
       "learning_rate": 2.4049079754601228e-05,
-      "loss": 1.3542532920837402,
+      "loss": 1.3542657852172852,
       "step": 50
     },
     {
       "epoch": 0.011086815928058883,
       "grad_norm": 0.76953125,
       "learning_rate": 2.895705521472393e-05,
-      "loss": 1.356162929534912,
+      "loss": 1.3561473846435548,
       "step": 60
     },
     {
       "epoch": 0.012934618582735364,
-      "grad_norm": 0.87890625,
+      "grad_norm": 0.875,
       "learning_rate": 3.386503067484663e-05,
-      "loss": 1.341750144958496,
+      "loss": 1.3417378425598145,
       "step": 70
     },
     {
       "epoch": 0.014782421237411844,
       "grad_norm": 0.83203125,
       "learning_rate": 3.8773006134969326e-05,
-      "loss": 1.3397028923034668,
+      "loss": 1.3396991729736327,
       "step": 80
     },
     {
       "epoch": 0.016630223892088326,
-      "grad_norm": 0.88671875,
+      "grad_norm": 0.8828125,
       "learning_rate": 4.368098159509202e-05,
-      "loss": 1.3383052825927735,
+      "loss": 1.3383003234863282,
       "step": 90
     },
     {
       "epoch": 0.018478026546764805,
-      "grad_norm": 0.86328125,
+      "grad_norm": 0.859375,
       "learning_rate": 4.858895705521473e-05,
-      "loss": 1.344656753540039,
+      "loss": 1.344669723510742,
       "step": 100
     },
     {
       "epoch": 0.020325829201441287,
-      "grad_norm": 0.890625,
+      "grad_norm": 0.88671875,
       "learning_rate": 5.349693251533743e-05,
-      "loss": 1.3360063552856445,
+      "loss": 1.3360023498535156,
       "step": 110
     },
     {
       "epoch": 0.022173631856117766,
       "grad_norm": 0.84765625,
       "learning_rate": 5.840490797546013e-05,
-      "loss": 1.333084487915039,
+      "loss": 1.3331090927124023,
       "step": 120
     },
     {
       "epoch": 0.02402143451079425,
-      "grad_norm": 1.0,
+      "grad_norm": 0.92578125,
       "learning_rate": 6.331288343558283e-05,
-      "loss": 1.3300580978393555,
+      "loss": 1.3300278663635254,
       "step": 130
     },
     {
       "epoch": 0.025869237165470727,
-      "grad_norm": 1.1640625,
+      "grad_norm": 0.921875,
       "learning_rate": 6.822085889570553e-05,
-      "loss": 1.355912971496582,
+      "loss": 1.356159782409668,
       "step": 140
     },
     {
       "epoch": 0.02771703982014721,
-      "grad_norm": 1.0,
+      "grad_norm": 1.2265625,
       "learning_rate": 7.312883435582824e-05,
-      "loss": 1.328594970703125,
+      "loss": 1.3295406341552733,
       "step": 150
     },
     {
       "epoch": 0.029564842474823688,
-      "grad_norm": 0.85546875,
+      "grad_norm": 0.94921875,
       "learning_rate": 7.803680981595093e-05,
-      "loss": 1.3297951698303223,
+      "loss": 1.3274627685546876,
       "step": 160
     },
     {
       "epoch": 0.03141264512950017,
-      "grad_norm": 0.83984375,
+      "grad_norm": 0.96875,
       "learning_rate": 7.999974208379523e-05,
-      "loss": 1.333768653869629,
+      "loss": 1.3343279838562012,
       "step": 170
     },
     {
       "epoch": 0.03326044778417665,
-      "grad_norm": 1.5546875,
+      "grad_norm": 1.03125,
       "learning_rate": 7.999816594125531e-05,
-      "loss": 1.339357566833496,
+      "loss": 1.3382596015930175,
       "step": 180
     },
     {
       "epoch": 0.03510825043885313,
-      "grad_norm": 0.99609375,
+      "grad_norm": 0.86328125,
       "learning_rate": 7.999515699934749e-05,
-      "loss": 1.3567001342773437,
+      "loss": 1.356815719604492,
       "step": 190
     },
     {
       "epoch": 0.03695605309352961,
-      "grad_norm": 1.171875,
+      "grad_norm": 0.94140625,
       "learning_rate": 7.999071536585692e-05,
-      "loss": 1.3331151962280274,
+      "loss": 1.3365495681762696,
       "step": 200
     },
     {
       "epoch": 0.03880385574820609,
-      "grad_norm": 0.91015625,
+      "grad_norm": 1.015625,
       "learning_rate": 7.998484119989019e-05,
-      "loss": 1.3421928405761718,
+      "loss": 1.345808219909668,
       "step": 210
     },
     {
       "epoch": 0.040651658402882575,
-      "grad_norm": 1.671875,
+      "grad_norm": 1.0390625,
       "learning_rate": 7.997753471186945e-05,
-      "loss": 1.3528425216674804,
+      "loss": 1.3547552108764649,
       "step": 220
     },
     {
       "epoch": 0.04249946105755905,
-      "grad_norm": 1.3203125,
+      "grad_norm": 0.90625,
       "learning_rate": 7.996879616352498e-05,
-      "loss": 1.3393315315246581,
+      "loss": 1.3373971939086915,
       "step": 230
     },
     {
       "epoch": 0.04434726371223553,
-      "grad_norm": 1.3046875,
+      "grad_norm": 0.765625,
       "learning_rate": 7.99586258678858e-05,
-      "loss": 1.3321426391601563,
+      "loss": 1.339411163330078,
       "step": 240
     },
     {
       "epoch": 0.046195066366912015,
-      "grad_norm": 1.140625,
+      "grad_norm": 0.9921875,
       "learning_rate": 7.994702418926845e-05,
-      "loss": 1.316728401184082,
+      "loss": 1.330860710144043,
       "step": 250
     },
     {
       "epoch": 0.0480428690215885,
-      "grad_norm": 1.2734375,
+      "grad_norm": 0.875,
       "learning_rate": 7.993399154326394e-05,
-      "loss": 1.2948928833007813,
+      "loss": 1.3104736328125,
       "step": 260
     },
     {
       "epoch": 0.04989067167626497,
-      "grad_norm": 1.1015625,
+      "grad_norm": 0.8125,
       "learning_rate": 7.991952839672283e-05,
-      "loss": 1.340465259552002,
+      "loss": 1.3543720245361328,
       "step": 270
     },
     {
       "epoch": 0.051738474330941454,
-      "grad_norm": 1.28125,
+      "grad_norm": 0.90234375,
       "learning_rate": 7.990363526773856e-05,
-      "loss": 1.311416244506836,
+      "loss": 1.323147964477539,
       "step": 280
     },
     {
       "epoch": 0.05358627698561794,
-      "grad_norm": 1.5,
+      "grad_norm": 0.828125,
       "learning_rate": 7.988631272562887e-05,
-      "loss": 1.3211885452270509,
+      "loss": 1.3300386428833009,
       "step": 290
     },
     {
       "epoch": 0.05543407964029442,
-      "grad_norm": 0.8515625,
+      "grad_norm": 0.98828125,
       "learning_rate": 7.986756139091539e-05,
-      "loss": 1.3198785781860352,
+      "loss": 1.3283294677734374,
       "step": 300
     },
     {
       "epoch": 0.057281882294970894,
       "grad_norm": 0.94140625,
       "learning_rate": 7.984738193530144e-05,
-      "loss": 1.3197151184082032,
+      "loss": 1.3237747192382812,
       "step": 310
     },
     {
       "epoch": 0.059129684949647376,
-      "grad_norm": 0.80078125,
+      "grad_norm": 0.83203125,
       "learning_rate": 7.982577508164794e-05,
-      "loss": 1.3121904373168944,
+      "loss": 1.3112046241760253,
       "step": 320
     },
     {
       "epoch": 0.06097748760432386,
-      "grad_norm": 0.84375,
+      "grad_norm": 0.8828125,
       "learning_rate": 7.980274160394754e-05,
-      "loss": 1.3363779067993165,
+      "loss": 1.3327028274536132,
       "step": 330
     },
     {
       "epoch": 0.06282529025900034,
-      "grad_norm": 0.875,
+      "grad_norm": 0.796875,
       "learning_rate": 7.977828232729686e-05,
-      "loss": 1.3312028884887694,
+      "loss": 1.326256561279297,
       "step": 340
     },
     {
       "epoch": 0.06467309291367682,
-      "grad_norm": 1.0703125,
+      "grad_norm": 0.78515625,
       "learning_rate": 7.975239812786701e-05,
-      "loss": 1.3153843879699707,
+      "loss": 1.3104028701782227,
       "step": 350
     },
     {
       "epoch": 0.0665208955683533,
-      "grad_norm": 0.83984375,
+      "grad_norm": 0.89453125,
       "learning_rate": 7.972508993287209e-05,
-      "loss": 1.3332480430603026,
+      "loss": 1.328328514099121,
       "step": 360
     },
     {
       "epoch": 0.06836869822302978,
       "grad_norm": 0.8125,
       "learning_rate": 7.969635872053607e-05,
-      "loss": 1.3310165405273438,
+      "loss": 1.326300811767578,
       "step": 370
     },
     {
       "epoch": 0.07021650087770626,
-      "grad_norm": 0.74609375,
+      "grad_norm": 0.7890625,
       "learning_rate": 7.966620552005774e-05,
-      "loss": 1.3240743637084962,
+      "loss": 1.318770980834961,
       "step": 380
     },
     {
       "epoch": 0.07206430353238275,
-      "grad_norm": 0.73046875,
+      "grad_norm": 0.72265625,
       "learning_rate": 7.963463141157376e-05,
-      "loss": 1.3202999114990235,
+      "loss": 1.314008903503418,
       "step": 390
     },
     {
       "epoch": 0.07391210618705922,
       "grad_norm": 0.80078125,
       "learning_rate": 7.960163752612006e-05,
-      "loss": 1.3233794212341308,
+      "loss": 1.317710781097412,
       "step": 400
     },
     {
       "epoch": 0.0757599088417357,
-      "grad_norm": 0.7578125,
+      "grad_norm": 0.76953125,
       "learning_rate": 7.95672250455913e-05,
-      "loss": 1.3085984230041503,
+      "loss": 1.3031733512878418,
       "step": 410
     },
     {
       "epoch": 0.07760771149641219,
-      "grad_norm": 0.765625,
+      "grad_norm": 0.953125,
       "learning_rate": 7.95313952026985e-05,
-      "loss": 1.3132389068603516,
+      "loss": 1.3082040786743163,
       "step": 420
     },
     {
       "epoch": 0.07945551415108866,
-      "grad_norm": 0.7578125,
+      "grad_norm": 0.69140625,
       "learning_rate": 7.949414928092491e-05,
-      "loss": 1.3101947784423829,
+      "loss": 1.303591537475586,
       "step": 430
     },
     {
       "epoch": 0.08130331680576515,
-      "grad_norm": 0.78515625,
+      "grad_norm": 0.74609375,
       "learning_rate": 7.945548861448006e-05,
-      "loss": 1.305581283569336,
+      "loss": 1.2992240905761718,
       "step": 440
     },
     {
       "epoch": 0.08315111946044162,
-      "grad_norm": 0.734375,
+      "grad_norm": 0.79296875,
       "learning_rate": 7.94154145882519e-05,
-      "loss": 1.3113672256469726,
+      "loss": 1.3058340072631835,
       "step": 450
     },
     {
       "epoch": 0.0849989221151181,
-      "grad_norm": 0.70703125,
+      "grad_norm": 0.74609375,
       "learning_rate": 7.937392863775721e-05,
-      "loss": 1.3072314262390137,
+      "loss": 1.3020547866821288,
       "step": 460
     },
     {
       "epoch": 0.08684672476979459,
       "grad_norm": 0.7109375,
       "learning_rate": 7.933103224909024e-05,
-      "loss": 1.2941503524780273,
+      "loss": 1.2893502235412597,
       "step": 470
     },
     {
       "epoch": 0.08869452742447106,
-      "grad_norm": 0.85546875,
+      "grad_norm": 0.83203125,
       "learning_rate": 7.928672695886943e-05,
-      "loss": 1.3035563468933105,
+      "loss": 1.2988996505737305,
       "step": 480
     },
     {
       "epoch": 0.09054233007914755,
-      "grad_norm": 0.73828125,
+      "grad_norm": 0.76171875,
       "learning_rate": 7.924101435418232e-05,
-      "loss": 1.3083011627197265,
+      "loss": 1.303721809387207,
       "step": 490
     },
     {
       "epoch": 0.09239013273382403,
-      "grad_norm": 0.76171875,
+      "grad_norm": 0.74609375,
       "learning_rate": 7.919389607252879e-05,
-      "loss": 1.3010213851928711,
+      "loss": 1.2966513633728027,
       "step": 500
     },
     {
       "epoch": 0.0942379353885005,
-      "grad_norm": 0.8515625,
+      "grad_norm": 0.7578125,
       "learning_rate": 7.91453738017623e-05,
-      "loss": 1.3147708892822265,
+      "loss": 1.3101999282836914,
       "step": 510
     },
     {
       "epoch": 0.096085738043177,
-      "grad_norm": 0.71484375,
+      "grad_norm": 0.6953125,
       "learning_rate": 7.909544928002955e-05,
-      "loss": 1.3136831283569337,
+      "loss": 1.3093395233154297,
       "step": 520
     },
     {
       "epoch": 0.09793354069785347,
-      "grad_norm": 0.7734375,
+      "grad_norm": 0.7265625,
       "learning_rate": 7.904412429570812e-05,
-      "loss": 1.295796012878418,
+      "loss": 1.2916431427001953,
       "step": 530
     },
     {
       "epoch": 0.09978134335252994,
-      "grad_norm": 0.74609375,
+      "grad_norm": 0.79296875,
       "learning_rate": 7.899140068734243e-05,
-      "loss": 1.3016599655151366,
+      "loss": 1.2977453231811524,
       "step": 540
     },
     {
       "epoch": 0.10162914600720643,
-      "grad_norm": 0.734375,
+      "grad_norm": 0.70703125,
       "learning_rate": 7.893728034357788e-05,
-      "loss": 1.2941709518432618,
+      "loss": 1.2903645515441895,
       "step": 550
     },
     {
       "epoch": 0.10347694866188291,
-      "grad_norm": 0.74609375,
+      "grad_norm": 1.0078125,
       "learning_rate": 7.888176520309324e-05,
-      "loss": 1.303166675567627,
+      "loss": 1.2998083114624024,
       "step": 560
     },
     {
       "epoch": 0.1053247513165594,
-      "grad_norm": 0.8359375,
+      "grad_norm": 1.5546875,
       "learning_rate": 7.882485725453114e-05,
-      "loss": 1.2963369369506836,
+      "loss": 1.293901538848877,
       "step": 570
     },
     {
       "epoch": 0.10717255397123587,
-      "grad_norm": 0.76171875,
+      "grad_norm": 0.96484375,
       "learning_rate": 7.876655853642685e-05,
-      "loss": 1.2998339653015136,
+      "loss": 1.2969616889953612,
       "step": 580
     },
     {
       "epoch": 0.10902035662591235,
-      "grad_norm": 1.0546875,
+      "grad_norm": 0.953125,
       "learning_rate": 7.870687113713534e-05,
-      "loss": 1.2977848052978516,
+      "loss": 1.2882954597473144,
       "step": 590
     },
     {
       "epoch": 0.11086815928058884,
-      "grad_norm": 0.8515625,
+      "grad_norm": 0.859375,
       "learning_rate": 7.864579719475628e-05,
-      "loss": 1.2969680786132813,
+      "loss": 1.2879480361938476,
       "step": 600
     },
     {
       "epoch": 0.11271596193526531,
-      "grad_norm": 1.1875,
+      "grad_norm": 1.65625,
       "learning_rate": 7.858333889705768e-05,
-      "loss": 1.2845346450805664,
+      "loss": 1.275766944885254,
       "step": 610
     },
     {
       "epoch": 0.11456376458994179,
-      "grad_norm": 0.7265625,
+      "grad_norm": 1.0078125,
       "learning_rate": 7.851949848139733e-05,
-      "loss": 1.280721378326416,
+      "loss": 1.2760635375976563,
       "step": 620
     },
     {
       "epoch": 0.11641156724461828,
-      "grad_norm": 0.73828125,
+      "grad_norm": 0.765625,
       "learning_rate": 7.845427823464278e-05,
-      "loss": 1.2810577392578124,
+      "loss": 1.2763057708740235,
       "step": 630
     },
     {
       "epoch": 0.11825936989929475,
-      "grad_norm": 0.87109375,
+      "grad_norm": 0.95703125,
       "learning_rate": 7.838768049308936e-05,
-      "loss": 1.2661373138427734,
+      "loss": 1.2617834091186524,
       "step": 640
     },
     {
       "epoch": 0.12010717255397124,
-      "grad_norm": 1.421875,
+      "grad_norm": 1.15625,
       "learning_rate": 7.831970764237652e-05,
-      "loss": 1.2931352615356446,
+      "loss": 1.284263801574707,
       "step": 650
     },
     {
       "epoch": 0.12195497520864772,
-      "grad_norm": 1.0078125,
+      "grad_norm": 0.890625,
       "learning_rate": 7.825036211740233e-05,
-      "loss": 1.2815824508666993,
+      "loss": 1.275103759765625,
       "step": 660
     },
     {
       "epoch": 0.12380277786332419,
-      "grad_norm": 0.796875,
+      "grad_norm": 0.76171875,
       "learning_rate": 7.817964640223627e-05,
-      "loss": 1.2806090354919433,
+      "loss": 1.2780118942260743,
       "step": 670
     },
     {
       "epoch": 0.12565058051800068,
-      "grad_norm": 0.8203125,
+      "grad_norm": 0.765625,
       "learning_rate": 7.810756303003034e-05,
-      "loss": 1.2836750030517579,
+      "loss": 1.2842129707336425,
       "step": 680
     },
     {
       "epoch": 0.12749838317267717,
-      "grad_norm": 1.046875,
+      "grad_norm": 0.83984375,
       "learning_rate": 7.803411458292815e-05,
-      "loss": 1.2836138725280761,
+      "loss": 1.2833771705627441,
       "step": 690
     },
     {
       "epoch": 0.12934618582735363,
-      "grad_norm": 0.82421875,
+      "grad_norm": 0.72265625,
       "learning_rate": 7.795930369197257e-05,
-      "loss": 1.2726373672485352,
+      "loss": 1.2730409622192382,
       "step": 700
     },
     {
       "epoch": 0.13119398848203012,
-      "grad_norm": 0.765625,
+      "grad_norm": 0.69140625,
       "learning_rate": 7.78831330370114e-05,
-      "loss": 1.2706066131591798,
+      "loss": 1.2730585098266602,
       "step": 710
     },
     {
       "epoch": 0.1330417911367066,
-      "grad_norm": 0.91796875,
+      "grad_norm": 0.8125,
       "learning_rate": 7.780560534660141e-05,
-      "loss": 1.2896896362304688,
+      "loss": 1.29056978225708,
       "step": 720
     },
     {
       "epoch": 0.13488959379138307,
-      "grad_norm": 0.8828125,
+      "grad_norm": 0.8046875,
       "learning_rate": 7.772672339791056e-05,
-      "loss": 1.2695566177368165,
+      "loss": 1.2694992065429687,
       "step": 730
     },
     {
       "epoch": 0.13673739644605956,
       "grad_norm": 0.69921875,
       "learning_rate": 7.764649001661861e-05,
-      "loss": 1.288385009765625,
+      "loss": 1.2878114700317382,
       "step": 740
     },
     {
       "epoch": 0.13858519910073605,
-      "grad_norm": 0.6953125,
+      "grad_norm": 0.77734375,
       "learning_rate": 7.756490807681575e-05,
-      "loss": 1.2738194465637207,
+      "loss": 1.2727023124694825,
       "step": 750
     },
     {
       "epoch": 0.1404330017554125,
-      "grad_norm": 0.6875,
+      "grad_norm": 0.71484375,
       "learning_rate": 7.748198050089979e-05,
-      "loss": 1.2750391006469726,
+      "loss": 1.2734295845031738,
       "step": 760
     },
     {
       "epoch": 0.142280804410089,
-      "grad_norm": 0.7421875,
+      "grad_norm": 0.97265625,
       "learning_rate": 7.739771025947141e-05,
-      "loss": 1.2766295433044434,
+      "loss": 1.2755632400512695,
       "step": 770
     },
     {
       "epoch": 0.1441286070647655,
-      "grad_norm": 0.69921875,
+      "grad_norm": 0.875,
       "learning_rate": 7.731210037122771e-05,
-      "loss": 1.2519859313964843,
+      "loss": 1.250746726989746,
       "step": 780
     },
     {
       "epoch": 0.14597640971944195,
-      "grad_norm": 0.69921875,
+      "grad_norm": 0.8046875,
       "learning_rate": 7.722515390285417e-05,
-      "loss": 1.277346420288086,
+      "loss": 1.2750632286071777,
       "step": 790
     },
     {
       "epoch": 0.14782421237411844,
-      "grad_norm": 0.734375,
+      "grad_norm": 0.80859375,
       "learning_rate": 7.71368739689147e-05,
-      "loss": 1.270993137359619,
+      "loss": 1.2690186500549316,
       "step": 800
     },
     {
       "epoch": 0.14967201502879493,
-      "grad_norm": 0.71875,
+      "grad_norm": 0.734375,
       "learning_rate": 7.704726373174014e-05,
-      "loss": 1.277150535583496,
+      "loss": 1.2751304626464843,
       "step": 810
     },
     {
       "epoch": 0.15004157555973022,
-      "eval_loss": 1.27553129196167,
-      "eval_runtime": 145.8255,
-      "eval_samples_per_second": 21.56,
-      "eval_steps_per_second": 5.39,
+      "eval_loss": 1.2738391160964966,
+      "eval_runtime": 145.532,
+      "eval_samples_per_second": 21.603,
+      "eval_steps_per_second": 5.401,
       "step": 812
     }
   ],