| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.0, |
| "eval_steps": 500, |
| "global_step": 1890, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.5192923663184046, |
| "epoch": 0.05293631100082713, |
| "grad_norm": 11.9375, |
| "learning_rate": 3.2142857142857147e-06, |
| "loss": 1.3518179893493651, |
| "mean_token_accuracy": 0.6921450330875814, |
| "num_tokens": 710093.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.0496994502842427, |
| "epoch": 0.10587262200165426, |
| "grad_norm": 3.109375, |
| "learning_rate": 6.785714285714287e-06, |
| "loss": 1.0432048797607423, |
| "mean_token_accuracy": 0.7421417662873864, |
| "num_tokens": 1429271.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.8430325930938125, |
| "epoch": 0.1588089330024814, |
| "grad_norm": 2.34375, |
| "learning_rate": 1.0357142857142859e-05, |
| "loss": 0.8576922416687012, |
| "mean_token_accuracy": 0.7759388316422701, |
| "num_tokens": 2149735.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.7610162339173258, |
| "epoch": 0.21174524400330852, |
| "grad_norm": 1.6015625, |
| "learning_rate": 1.3928571428571429e-05, |
| "loss": 0.7648541927337646, |
| "mean_token_accuracy": 0.795621344819665, |
| "num_tokens": 2868867.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.724903151486069, |
| "epoch": 0.26468155500413565, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.7500000000000002e-05, |
| "loss": 0.7256669044494629, |
| "mean_token_accuracy": 0.8042738335207105, |
| "num_tokens": 3582476.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.6910091611556709, |
| "epoch": 0.3176178660049628, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.999986795783372e-05, |
| "loss": 0.6930101394653321, |
| "mean_token_accuracy": 0.8116411617025733, |
| "num_tokens": 4287689.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.6818584973923862, |
| "epoch": 0.3705541770057899, |
| "grad_norm": 1.4375, |
| "learning_rate": 1.9997520638548385e-05, |
| "loss": 0.6860392570495606, |
| "mean_token_accuracy": 0.8126627463847399, |
| "num_tokens": 5011191.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.6760601634159684, |
| "epoch": 0.42349048800661704, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.9992239841688418e-05, |
| "loss": 0.6755449295043945, |
| "mean_token_accuracy": 0.8157186867669225, |
| "num_tokens": 5714842.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.6550427805632353, |
| "epoch": 0.47642679900744417, |
| "grad_norm": 1.2890625, |
| "learning_rate": 1.9984027116747914e-05, |
| "loss": 0.6541121959686279, |
| "mean_token_accuracy": 0.8201886491850019, |
| "num_tokens": 6424614.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.6484816522337497, |
| "epoch": 0.5293631100082713, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.997288487350878e-05, |
| "loss": 0.6504482746124267, |
| "mean_token_accuracy": 0.8210024476051331, |
| "num_tokens": 7139778.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.6473308478482067, |
| "epoch": 0.5822994210090985, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.9958816381333684e-05, |
| "loss": 0.6473936557769775, |
| "mean_token_accuracy": 0.8223220122978091, |
| "num_tokens": 7852540.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.6404711910523474, |
| "epoch": 0.6352357320099256, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.994182576820673e-05, |
| "loss": 0.6405678272247315, |
| "mean_token_accuracy": 0.8234121534973383, |
| "num_tokens": 8568056.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.6317982953041792, |
| "epoch": 0.6881720430107527, |
| "grad_norm": 1.25, |
| "learning_rate": 1.9921918019522238e-05, |
| "loss": 0.6368978500366211, |
| "mean_token_accuracy": 0.8247422102838755, |
| "num_tokens": 9276742.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.631435402482748, |
| "epoch": 0.7411083540115798, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.9899098976621922e-05, |
| "loss": 0.6334238052368164, |
| "mean_token_accuracy": 0.8254026060923934, |
| "num_tokens": 9990163.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.6324203921481967, |
| "epoch": 0.794044665012407, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.9873375335080915e-05, |
| "loss": 0.629639720916748, |
| "mean_token_accuracy": 0.8260580543428659, |
| "num_tokens": 10710006.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.624595766980201, |
| "epoch": 0.8469809760132341, |
| "grad_norm": 1.25, |
| "learning_rate": 1.9844754642743164e-05, |
| "loss": 0.6285017013549805, |
| "mean_token_accuracy": 0.8260432455688715, |
| "num_tokens": 11416822.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.6298610974103213, |
| "epoch": 0.8999172870140613, |
| "grad_norm": 1.25, |
| "learning_rate": 1.981324529750672e-05, |
| "loss": 0.6308236598968506, |
| "mean_token_accuracy": 0.8258236780762672, |
| "num_tokens": 12123118.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.6201933478936553, |
| "epoch": 0.9528535980148883, |
| "grad_norm": 1.25, |
| "learning_rate": 1.9778856544859647e-05, |
| "loss": 0.6202278137207031, |
| "mean_token_accuracy": 0.8279775837436318, |
| "num_tokens": 12831058.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.6077935299850789, |
| "epoch": 1.0052936311000826, |
| "grad_norm": 1.2578125, |
| "learning_rate": 1.9741598475167173e-05, |
| "loss": 0.6005611419677734, |
| "mean_token_accuracy": 0.8322159181633958, |
| "num_tokens": 13539870.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.5737332610413433, |
| "epoch": 1.0582299421009098, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.9701482020711004e-05, |
| "loss": 0.5751582622528076, |
| "mean_token_accuracy": 0.8365032711997629, |
| "num_tokens": 14256699.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.5619327845051885, |
| "epoch": 1.111166253101737, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.9658518952481543e-05, |
| "loss": 0.5628788948059082, |
| "mean_token_accuracy": 0.8406051754951477, |
| "num_tokens": 14977492.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.5730686764232814, |
| "epoch": 1.1641025641025642, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.9612721876724054e-05, |
| "loss": 0.5720590591430664, |
| "mean_token_accuracy": 0.8386186562478543, |
| "num_tokens": 15693723.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.5607471418567002, |
| "epoch": 1.2170388751033911, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.9564104231239726e-05, |
| "loss": 0.5599365234375, |
| "mean_token_accuracy": 0.8400625573471189, |
| "num_tokens": 16404449.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.5628137235529721, |
| "epoch": 1.2699751861042183, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.9512680281442745e-05, |
| "loss": 0.5613645076751709, |
| "mean_token_accuracy": 0.8405855750665069, |
| "num_tokens": 17118450.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.5673876556567847, |
| "epoch": 1.3229114971050455, |
| "grad_norm": 1.2109375, |
| "learning_rate": 1.9458465116174534e-05, |
| "loss": 0.5634001731872559, |
| "mean_token_accuracy": 0.840095579251647, |
| "num_tokens": 17826861.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.5628660793416203, |
| "epoch": 1.3758478081058727, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.940147464327637e-05, |
| "loss": 0.5646507263183593, |
| "mean_token_accuracy": 0.8398628858849406, |
| "num_tokens": 18545340.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.5588320679031312, |
| "epoch": 1.4287841191067, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.9341725584921712e-05, |
| "loss": 0.55802321434021, |
| "mean_token_accuracy": 0.8421088399365544, |
| "num_tokens": 19262118.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.560073831025511, |
| "epoch": 1.4817204301075269, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.9279235472709554e-05, |
| "loss": 0.5612293243408203, |
| "mean_token_accuracy": 0.8408227931708098, |
| "num_tokens": 19968479.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.5694392369128763, |
| "epoch": 1.534656741108354, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.92140226425203e-05, |
| "loss": 0.566476583480835, |
| "mean_token_accuracy": 0.8386643555015325, |
| "num_tokens": 20668540.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.5653179603628814, |
| "epoch": 1.587593052109181, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.914610622913566e-05, |
| "loss": 0.5663981437683105, |
| "mean_token_accuracy": 0.8399667147547006, |
| "num_tokens": 21377905.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.556015994027257, |
| "epoch": 1.6405293631100082, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.9075506160624068e-05, |
| "loss": 0.5562931060791015, |
| "mean_token_accuracy": 0.8419139498844743, |
| "num_tokens": 22086263.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.5500682834535837, |
| "epoch": 1.6934656741108354, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.900224315249343e-05, |
| "loss": 0.5473555088043213, |
| "mean_token_accuracy": 0.8436701815575361, |
| "num_tokens": 22790833.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.5480125053785742, |
| "epoch": 1.7464019851116626, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.8926338701612738e-05, |
| "loss": 0.546162748336792, |
| "mean_token_accuracy": 0.8438417742028832, |
| "num_tokens": 23503166.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.5573178362101316, |
| "epoch": 1.7993382961124897, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.884781507990445e-05, |
| "loss": 0.5579123497009277, |
| "mean_token_accuracy": 0.8416105538606644, |
| "num_tokens": 24223112.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.5548438420519233, |
| "epoch": 1.852274607113317, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.8766695327809463e-05, |
| "loss": 0.552967643737793, |
| "mean_token_accuracy": 0.8425577716901899, |
| "num_tokens": 24936212.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.5425214732997119, |
| "epoch": 1.905210918114144, |
| "grad_norm": 1.125, |
| "learning_rate": 1.8683003247526567e-05, |
| "loss": 0.5432563304901123, |
| "mean_token_accuracy": 0.8448027539998293, |
| "num_tokens": 25662368.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.5548607715405524, |
| "epoch": 1.958147229114971, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.8596763396028384e-05, |
| "loss": 0.5571085929870605, |
| "mean_token_accuracy": 0.842253121547401, |
| "num_tokens": 26371713.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.5529434020963001, |
| "epoch": 2.0105872622001653, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.8508001077855876e-05, |
| "loss": 0.5430344581604004, |
| "mean_token_accuracy": 0.845259289072139, |
| "num_tokens": 27082791.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.48950026873499153, |
| "epoch": 2.0635235732009924, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.8416742337693436e-05, |
| "loss": 0.49013280868530273, |
| "mean_token_accuracy": 0.8564883340150118, |
| "num_tokens": 27804616.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.5000267030671239, |
| "epoch": 2.1164598842018196, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.8323013952726877e-05, |
| "loss": 0.4983365058898926, |
| "mean_token_accuracy": 0.8543869256973267, |
| "num_tokens": 28510447.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.49870071215555073, |
| "epoch": 2.169396195202647, |
| "grad_norm": 1.25, |
| "learning_rate": 1.8226843424786434e-05, |
| "loss": 0.49672441482543944, |
| "mean_token_accuracy": 0.854990890994668, |
| "num_tokens": 29231259.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.4919801990501583, |
| "epoch": 2.222332506203474, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.8128258972277176e-05, |
| "loss": 0.488813591003418, |
| "mean_token_accuracy": 0.8567689899355173, |
| "num_tokens": 29952760.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.5022821069695056, |
| "epoch": 2.275268817204301, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.8027289521899143e-05, |
| "loss": 0.5026164054870605, |
| "mean_token_accuracy": 0.8535820486024022, |
| "num_tokens": 30665899.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.5073185523040593, |
| "epoch": 2.3282051282051284, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.7923964700159658e-05, |
| "loss": 0.5066654205322265, |
| "mean_token_accuracy": 0.8529624160379171, |
| "num_tokens": 31372484.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.5107574328780174, |
| "epoch": 2.3811414392059556, |
| "grad_norm": 1.25, |
| "learning_rate": 1.78183148246803e-05, |
| "loss": 0.5092263221740723, |
| "mean_token_accuracy": 0.8523802844807505, |
| "num_tokens": 32096632.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.5019051403738558, |
| "epoch": 2.4340777502067823, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.77103708953011e-05, |
| "loss": 0.500576114654541, |
| "mean_token_accuracy": 0.8538923777639866, |
| "num_tokens": 32804805.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.5081322279758751, |
| "epoch": 2.4870140612076095, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.7600164584984548e-05, |
| "loss": 0.5055245399475098, |
| "mean_token_accuracy": 0.8527956234291196, |
| "num_tokens": 33508738.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.4976038018241525, |
| "epoch": 2.5399503722084367, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.7487728230522085e-05, |
| "loss": 0.4950095176696777, |
| "mean_token_accuracy": 0.8554171722382307, |
| "num_tokens": 34225982.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.5060889236629009, |
| "epoch": 2.592886683209264, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.737309482304585e-05, |
| "loss": 0.5059161186218262, |
| "mean_token_accuracy": 0.8531035153195262, |
| "num_tokens": 34943657.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.5023137603886425, |
| "epoch": 2.645822994210091, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.7256297998348388e-05, |
| "loss": 0.501007080078125, |
| "mean_token_accuracy": 0.85395945943892, |
| "num_tokens": 35644283.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.501791013032198, |
| "epoch": 2.6987593052109182, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.7137372027013212e-05, |
| "loss": 0.5015331268310547, |
| "mean_token_accuracy": 0.8534267192706466, |
| "num_tokens": 36350379.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.5038612156175077, |
| "epoch": 2.7516956162117454, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.7016351804359134e-05, |
| "loss": 0.5033885955810546, |
| "mean_token_accuracy": 0.8544643085449934, |
| "num_tokens": 37060303.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.5093710655346513, |
| "epoch": 2.804631927212572, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.6893272840201243e-05, |
| "loss": 0.5068147659301758, |
| "mean_token_accuracy": 0.8531381724402308, |
| "num_tokens": 37772753.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.49741906598210334, |
| "epoch": 2.8575682382134, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.6768171248431603e-05, |
| "loss": 0.49800939559936525, |
| "mean_token_accuracy": 0.8552119933068753, |
| "num_tokens": 38489638.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.495670201908797, |
| "epoch": 2.9105045492142265, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.66410837364227e-05, |
| "loss": 0.493875789642334, |
| "mean_token_accuracy": 0.8558088952675462, |
| "num_tokens": 39198407.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.502962542604655, |
| "epoch": 2.9634408602150537, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.651204759425674e-05, |
| "loss": 0.49970645904541017, |
| "mean_token_accuracy": 0.8544731307774782, |
| "num_tokens": 39917572.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.49963319781074766, |
| "epoch": 3.015880893300248, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.6381100683783984e-05, |
| "loss": 0.4904365062713623, |
| "mean_token_accuracy": 0.8567472978345227, |
| "num_tokens": 40623043.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 0.46208551060408354, |
| "epoch": 3.0688172043010753, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.6248281427513293e-05, |
| "loss": 0.464261531829834, |
| "mean_token_accuracy": 0.8622843077406287, |
| "num_tokens": 41339395.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.46556579079478977, |
| "epoch": 3.1217535153019025, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.6113628797338182e-05, |
| "loss": 0.46287012100219727, |
| "mean_token_accuracy": 0.8632426928728819, |
| "num_tokens": 42048417.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.45391426915302874, |
| "epoch": 3.1746898263027297, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.597718230310167e-05, |
| "loss": 0.44995770454406736, |
| "mean_token_accuracy": 0.8659271128475666, |
| "num_tokens": 42764993.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.4522254983894527, |
| "epoch": 3.227626137303557, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.5838981981003273e-05, |
| "loss": 0.4518561363220215, |
| "mean_token_accuracy": 0.8663289973512291, |
| "num_tokens": 43484113.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 0.4578624659217894, |
| "epoch": 3.2805624483043836, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.5699068381851576e-05, |
| "loss": 0.457172155380249, |
| "mean_token_accuracy": 0.8645018568262458, |
| "num_tokens": 44190080.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 0.4634090135805309, |
| "epoch": 3.333498759305211, |
| "grad_norm": 1.25, |
| "learning_rate": 1.5557482559165777e-05, |
| "loss": 0.46218481063842776, |
| "mean_token_accuracy": 0.8634107686579228, |
| "num_tokens": 44901717.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.4552931568585336, |
| "epoch": 3.386435070306038, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.541426605712975e-05, |
| "loss": 0.45178699493408203, |
| "mean_token_accuracy": 0.8655921226367355, |
| "num_tokens": 45611553.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.4637038384564221, |
| "epoch": 3.439371381306865, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.526946089840214e-05, |
| "loss": 0.46104907989501953, |
| "mean_token_accuracy": 0.8630832064896822, |
| "num_tokens": 46319422.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.4613310274202377, |
| "epoch": 3.4923076923076923, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.5123109571786034e-05, |
| "loss": 0.46049203872680666, |
| "mean_token_accuracy": 0.8640554174780846, |
| "num_tokens": 47040011.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.46247419929131867, |
| "epoch": 3.5452440033085195, |
| "grad_norm": 1.2890625, |
| "learning_rate": 1.4975255019761896e-05, |
| "loss": 0.46044178009033204, |
| "mean_token_accuracy": 0.8631454292684794, |
| "num_tokens": 47747908.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.46336479745805265, |
| "epoch": 3.5981803143093467, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.4825940625887343e-05, |
| "loss": 0.4608856201171875, |
| "mean_token_accuracy": 0.8629281781613827, |
| "num_tokens": 48465834.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.4616638246923685, |
| "epoch": 3.6511166253101734, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.467521020206753e-05, |
| "loss": 0.4600059509277344, |
| "mean_token_accuracy": 0.8631995480507613, |
| "num_tokens": 49175962.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.46175111047923567, |
| "epoch": 3.704052936311001, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.4523107975699833e-05, |
| "loss": 0.45844330787658694, |
| "mean_token_accuracy": 0.8639702210202813, |
| "num_tokens": 49891839.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.4650504776276648, |
| "epoch": 3.756989247311828, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.4369678576696633e-05, |
| "loss": 0.46411809921264646, |
| "mean_token_accuracy": 0.8627843948081135, |
| "num_tokens": 50614080.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.45294095706194637, |
| "epoch": 3.809925558312655, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.4214967024389978e-05, |
| "loss": 0.4501089096069336, |
| "mean_token_accuracy": 0.865597159229219, |
| "num_tokens": 51323270.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.4626351878978312, |
| "epoch": 3.862861869313482, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.4059018714321986e-05, |
| "loss": 0.46165976524353025, |
| "mean_token_accuracy": 0.8630725203081966, |
| "num_tokens": 52026181.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.45373933035880326, |
| "epoch": 3.9157981803143094, |
| "grad_norm": 1.2890625, |
| "learning_rate": 1.390187940492486e-05, |
| "loss": 0.45296554565429686, |
| "mean_token_accuracy": 0.8657654097303749, |
| "num_tokens": 52734953.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.45326819689944386, |
| "epoch": 3.9687344913151366, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.3743595204094442e-05, |
| "loss": 0.4504420280456543, |
| "mean_token_accuracy": 0.8658560564741492, |
| "num_tokens": 53453384.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.4462546736754077, |
| "epoch": 4.0211745244003305, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.3584212555661187e-05, |
| "loss": 0.43608269691467283, |
| "mean_token_accuracy": 0.869793642206523, |
| "num_tokens": 54163412.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.423415100434795, |
| "epoch": 4.074110835401158, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.3423778225762609e-05, |
| "loss": 0.42355055809020997, |
| "mean_token_accuracy": 0.8724647520110012, |
| "num_tokens": 54870856.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.4298218970187008, |
| "epoch": 4.127047146401985, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.3262339289121129e-05, |
| "loss": 0.42810845375061035, |
| "mean_token_accuracy": 0.8713828312233091, |
| "num_tokens": 55582308.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.43030765345320104, |
| "epoch": 4.1799834574028125, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.3099943115231386e-05, |
| "loss": 0.4289811611175537, |
| "mean_token_accuracy": 0.8711980657652021, |
| "num_tokens": 56302219.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.4318474263884127, |
| "epoch": 4.232919768403639, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.2936637354461064e-05, |
| "loss": 0.4266523838043213, |
| "mean_token_accuracy": 0.8714847691357136, |
| "num_tokens": 57015624.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.4312989429105073, |
| "epoch": 4.285856079404467, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.2772469924069285e-05, |
| "loss": 0.4299038887023926, |
| "mean_token_accuracy": 0.8718423148617148, |
| "num_tokens": 57728689.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.422397783678025, |
| "epoch": 4.338792390405294, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.2607488994146703e-05, |
| "loss": 0.4201834678649902, |
| "mean_token_accuracy": 0.8738429831340909, |
| "num_tokens": 58440887.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.42779573444277047, |
| "epoch": 4.39172870140612, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.2441742973481432e-05, |
| "loss": 0.42258238792419434, |
| "mean_token_accuracy": 0.8721055008471013, |
| "num_tokens": 59161499.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.4218779653310776, |
| "epoch": 4.444665012406948, |
| "grad_norm": 1.25, |
| "learning_rate": 1.2275280495354889e-05, |
| "loss": 0.41907758712768556, |
| "mean_token_accuracy": 0.8739795790985226, |
| "num_tokens": 59872509.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.4264325203374028, |
| "epoch": 4.497601323407775, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.21081504032718e-05, |
| "loss": 0.42411088943481445, |
| "mean_token_accuracy": 0.8730311194434762, |
| "num_tokens": 60593833.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.42463446664623916, |
| "epoch": 4.550537634408602, |
| "grad_norm": 1.25, |
| "learning_rate": 1.1940401736628524e-05, |
| "loss": 0.4204378128051758, |
| "mean_token_accuracy": 0.8726064411923289, |
| "num_tokens": 61303595.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.42604947271756827, |
| "epoch": 4.603473945409429, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.1772083716323878e-05, |
| "loss": 0.4234048843383789, |
| "mean_token_accuracy": 0.8727520015090704, |
| "num_tokens": 62014565.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.4270178693346679, |
| "epoch": 4.656410256410257, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.1603245730316724e-05, |
| "loss": 0.4217417240142822, |
| "mean_token_accuracy": 0.8718522403389215, |
| "num_tokens": 62723366.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.4276386719197035, |
| "epoch": 4.7093465674110835, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.1433937319134512e-05, |
| "loss": 0.4275094985961914, |
| "mean_token_accuracy": 0.8715818202123046, |
| "num_tokens": 63432928.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.4229441821575165, |
| "epoch": 4.762282878411911, |
| "grad_norm": 1.2578125, |
| "learning_rate": 1.1264208161337091e-05, |
| "loss": 0.42129974365234374, |
| "mean_token_accuracy": 0.8729970380663872, |
| "num_tokens": 64137723.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.43000464104115965, |
| "epoch": 4.815219189412738, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.1094108058940005e-05, |
| "loss": 0.42574424743652345, |
| "mean_token_accuracy": 0.8717775771394372, |
| "num_tokens": 64851083.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.4233045262284577, |
| "epoch": 4.868155500413565, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.0923686922801537e-05, |
| "loss": 0.4198883533477783, |
| "mean_token_accuracy": 0.8735994193702936, |
| "num_tokens": 65572798.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.42122035566717386, |
| "epoch": 4.921091811414392, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.0752994757977862e-05, |
| "loss": 0.41748809814453125, |
| "mean_token_accuracy": 0.8742743641138077, |
| "num_tokens": 66285398.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.42782136364839973, |
| "epoch": 4.974028122415219, |
| "grad_norm": 1.2109375, |
| "learning_rate": 1.058208164905052e-05, |
| "loss": 0.4283552646636963, |
| "mean_token_accuracy": 0.8719127237796783, |
| "num_tokens": 66999572.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.4207373881941714, |
| "epoch": 5.026468155500414, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.041099774543059e-05, |
| "loss": 0.410534143447876, |
| "mean_token_accuracy": 0.8757580184033993, |
| "num_tokens": 67710110.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.39997034817934035, |
| "epoch": 5.079404466501241, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.023979324664382e-05, |
| "loss": 0.3979193687438965, |
| "mean_token_accuracy": 0.8784487284719944, |
| "num_tokens": 68425350.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.4020651245024055, |
| "epoch": 5.132340777502068, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.0068518387601038e-05, |
| "loss": 0.3978609085083008, |
| "mean_token_accuracy": 0.878536625765264, |
| "num_tokens": 69142159.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.39861540934070944, |
| "epoch": 5.185277088502895, |
| "grad_norm": 1.2734375, |
| "learning_rate": 9.897223423858226e-06, |
| "loss": 0.39521949291229247, |
| "mean_token_accuracy": 0.87957021612674, |
| "num_tokens": 69854650.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.4008810735307634, |
| "epoch": 5.238213399503722, |
| "grad_norm": 1.2421875, |
| "learning_rate": 9.725958616870495e-06, |
| "loss": 0.397627592086792, |
| "mean_token_accuracy": 0.8790520671755075, |
| "num_tokens": 70570857.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.4153812664095312, |
| "epoch": 5.291149710504549, |
| "grad_norm": 1.28125, |
| "learning_rate": 9.554774219244344e-06, |
| "loss": 0.4131373405456543, |
| "mean_token_accuracy": 0.8753986166790128, |
| "num_tokens": 71287442.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.40678592287003995, |
| "epoch": 5.344086021505376, |
| "grad_norm": 1.171875, |
| "learning_rate": 9.38372045999252e-06, |
| "loss": 0.4011257171630859, |
| "mean_token_accuracy": 0.8782388668507337, |
| "num_tokens": 72007448.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.4018311721738428, |
| "epoch": 5.397022332506204, |
| "grad_norm": 1.296875, |
| "learning_rate": 9.212847529795784e-06, |
| "loss": 0.40113091468811035, |
| "mean_token_accuracy": 0.8777982145547867, |
| "num_tokens": 72715352.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.4099262719973922, |
| "epoch": 5.44995864350703, |
| "grad_norm": 1.2421875, |
| "learning_rate": 9.042205566275952e-06, |
| "loss": 0.40550937652587893, |
| "mean_token_accuracy": 0.877083626575768, |
| "num_tokens": 73429166.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.41102300900965927, |
| "epoch": 5.502894954507858, |
| "grad_norm": 1.2421875, |
| "learning_rate": 8.871844639284479e-06, |
| "loss": 0.40628976821899415, |
| "mean_token_accuracy": 0.8766557598486543, |
| "num_tokens": 74133406.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.4129321951419115, |
| "epoch": 5.555831265508685, |
| "grad_norm": 1.1953125, |
| "learning_rate": 8.701814736210933e-06, |
| "loss": 0.41147375106811523, |
| "mean_token_accuracy": 0.8755610020831227, |
| "num_tokens": 74836672.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.40460854438133537, |
| "epoch": 5.608767576509512, |
| "grad_norm": 1.2578125, |
| "learning_rate": 8.532165747315686e-06, |
| "loss": 0.40326995849609376, |
| "mean_token_accuracy": 0.8770574113354087, |
| "num_tokens": 75557055.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.4103983862325549, |
| "epoch": 5.661703887510339, |
| "grad_norm": 1.25, |
| "learning_rate": 8.362947451091072e-06, |
| "loss": 0.40504894256591795, |
| "mean_token_accuracy": 0.8777938289567828, |
| "num_tokens": 76272061.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.40608077771030365, |
| "epoch": 5.714640198511166, |
| "grad_norm": 1.25, |
| "learning_rate": 8.194209499655364e-06, |
| "loss": 0.40426006317138674, |
| "mean_token_accuracy": 0.8773980582132935, |
| "num_tokens": 76989852.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 0.40846858704462646, |
| "epoch": 5.7675765095119935, |
| "grad_norm": 1.328125, |
| "learning_rate": 8.026001404183831e-06, |
| "loss": 0.4043857574462891, |
| "mean_token_accuracy": 0.877496475353837, |
| "num_tokens": 77687094.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.4118567893281579, |
| "epoch": 5.82051282051282, |
| "grad_norm": 1.25, |
| "learning_rate": 7.85837252038112e-06, |
| "loss": 0.4126093864440918, |
| "mean_token_accuracy": 0.8756023030728102, |
| "num_tokens": 78397944.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.4146808403078467, |
| "epoch": 5.873449131513648, |
| "grad_norm": 1.2578125, |
| "learning_rate": 7.69137203399931e-06, |
| "loss": 0.4076550006866455, |
| "mean_token_accuracy": 0.8766046687960625, |
| "num_tokens": 79107666.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.4010618781670928, |
| "epoch": 5.926385442514475, |
| "grad_norm": 1.203125, |
| "learning_rate": 7.525048946405791e-06, |
| "loss": 0.4005697250366211, |
| "mean_token_accuracy": 0.8790890416130424, |
| "num_tokens": 79824629.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 0.40944934943690897, |
| "epoch": 5.979321753515302, |
| "grad_norm": 1.25, |
| "learning_rate": 7.359452060205245e-06, |
| "loss": 0.4070614814758301, |
| "mean_token_accuracy": 0.8767168298363686, |
| "num_tokens": 80532805.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 0.4028015411708633, |
| "epoch": 6.031761786600496, |
| "grad_norm": 1.2890625, |
| "learning_rate": 7.194629964919989e-06, |
| "loss": 0.3959317684173584, |
| "mean_token_accuracy": 0.8795143580211073, |
| "num_tokens": 81243632.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.38516616756096483, |
| "epoch": 6.084698097601324, |
| "grad_norm": 1.3046875, |
| "learning_rate": 7.030631022732781e-06, |
| "loss": 0.3816915273666382, |
| "mean_token_accuracy": 0.8833343693986535, |
| "num_tokens": 81955824.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.3961218709126115, |
| "epoch": 6.137634408602151, |
| "grad_norm": 1.3125, |
| "learning_rate": 6.867503354296392e-06, |
| "loss": 0.39214468002319336, |
| "mean_token_accuracy": 0.8803520953282714, |
| "num_tokens": 82672365.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 0.40226894784718753, |
| "epoch": 6.190570719602977, |
| "grad_norm": 1.3203125, |
| "learning_rate": 6.705294824614004e-06, |
| "loss": 0.39856839179992676, |
| "mean_token_accuracy": 0.8786789864301682, |
| "num_tokens": 83385723.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.3966492464300245, |
| "epoch": 6.243507030603805, |
| "grad_norm": 1.1953125, |
| "learning_rate": 6.54405302899463e-06, |
| "loss": 0.39055469036102297, |
| "mean_token_accuracy": 0.8807535348460078, |
| "num_tokens": 84097235.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.39696404109708966, |
| "epoch": 6.296443341604632, |
| "grad_norm": 1.203125, |
| "learning_rate": 6.383825279087699e-06, |
| "loss": 0.3929750919342041, |
| "mean_token_accuracy": 0.8800927491858601, |
| "num_tokens": 84809815.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.3934732445981354, |
| "epoch": 6.349379652605459, |
| "grad_norm": 1.296875, |
| "learning_rate": 6.2246585890008245e-06, |
| "loss": 0.39183971881866453, |
| "mean_token_accuracy": 0.8811883484944701, |
| "num_tokens": 85522179.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.3892004740424454, |
| "epoch": 6.402315963606286, |
| "grad_norm": 1.296875, |
| "learning_rate": 6.06659966150491e-06, |
| "loss": 0.38753180503845214, |
| "mean_token_accuracy": 0.8817478435114026, |
| "num_tokens": 86246555.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.39504086719825865, |
| "epoch": 6.455252274607114, |
| "grad_norm": 1.2421875, |
| "learning_rate": 5.909694874330601e-06, |
| "loss": 0.3885345458984375, |
| "mean_token_accuracy": 0.8820378331467509, |
| "num_tokens": 86962128.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.39277244443073867, |
| "epoch": 6.5081885856079404, |
| "grad_norm": 1.2578125, |
| "learning_rate": 5.753990266560105e-06, |
| "loss": 0.389221715927124, |
| "mean_token_accuracy": 0.8815512251108885, |
| "num_tokens": 87674788.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 0.39534553145058454, |
| "epoch": 6.561124896608767, |
| "grad_norm": 1.265625, |
| "learning_rate": 5.599531525118373e-06, |
| "loss": 0.39423694610595705, |
| "mean_token_accuracy": 0.8799041228368878, |
| "num_tokens": 88377891.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.40004007569514216, |
| "epoch": 6.614061207609595, |
| "grad_norm": 1.265625, |
| "learning_rate": 5.446363971367638e-06, |
| "loss": 0.396582818031311, |
| "mean_token_accuracy": 0.8796247195452451, |
| "num_tokens": 89083739.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.4012724973261356, |
| "epoch": 6.666997518610422, |
| "grad_norm": 1.2578125, |
| "learning_rate": 5.29453254780917e-06, |
| "loss": 0.3958230972290039, |
| "mean_token_accuracy": 0.8796050645411014, |
| "num_tokens": 89799048.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 0.3932869556359947, |
| "epoch": 6.719933829611249, |
| "grad_norm": 1.2265625, |
| "learning_rate": 5.1440818048962575e-06, |
| "loss": 0.38956871032714846, |
| "mean_token_accuracy": 0.8809968201443553, |
| "num_tokens": 90506283.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 0.3971431264653802, |
| "epoch": 6.772870140612076, |
| "grad_norm": 1.3046875, |
| "learning_rate": 4.995055887962167e-06, |
| "loss": 0.3939245939254761, |
| "mean_token_accuracy": 0.8801254570484162, |
| "num_tokens": 91217518.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 0.3942943840753287, |
| "epoch": 6.825806451612904, |
| "grad_norm": 1.3046875, |
| "learning_rate": 4.8474985242669965e-06, |
| "loss": 0.390548038482666, |
| "mean_token_accuracy": 0.8807255454361439, |
| "num_tokens": 91921292.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 0.4053244000300765, |
| "epoch": 6.87874276261373, |
| "grad_norm": 1.2890625, |
| "learning_rate": 4.701453010167217e-06, |
| "loss": 0.4019917011260986, |
| "mean_token_accuracy": 0.8788083763793111, |
| "num_tokens": 92637986.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.3981741506140679, |
| "epoch": 6.931679073614557, |
| "grad_norm": 1.2734375, |
| "learning_rate": 4.556962198411631e-06, |
| "loss": 0.39420528411865235, |
| "mean_token_accuracy": 0.8803565550595522, |
| "num_tokens": 93350635.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 0.3841029854491353, |
| "epoch": 6.984615384615385, |
| "grad_norm": 1.2734375, |
| "learning_rate": 4.414068485567473e-06, |
| "loss": 0.38068561553955077, |
| "mean_token_accuracy": 0.8828032523393631, |
| "num_tokens": 94067855.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 0.38967673692041394, |
| "epoch": 7.037055417700579, |
| "grad_norm": 1.1796875, |
| "learning_rate": 4.272813799580417e-06, |
| "loss": 0.3833909511566162, |
| "mean_token_accuracy": 0.8831083998319103, |
| "num_tokens": 94783200.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 0.3877631796523929, |
| "epoch": 7.089991728701406, |
| "grad_norm": 1.265625, |
| "learning_rate": 4.1332395874720474e-06, |
| "loss": 0.3814324617385864, |
| "mean_token_accuracy": 0.8834948791190982, |
| "num_tokens": 95493882.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.38117631543427705, |
| "epoch": 7.142928039702233, |
| "grad_norm": 1.2109375, |
| "learning_rate": 3.9953868031784415e-06, |
| "loss": 0.37805852890014646, |
| "mean_token_accuracy": 0.8838670440018177, |
| "num_tokens": 96208176.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.38905520508997143, |
| "epoch": 7.195864350703061, |
| "grad_norm": 1.21875, |
| "learning_rate": 3.8592958955334636e-06, |
| "loss": 0.3866551399230957, |
| "mean_token_accuracy": 0.8819786906242371, |
| "num_tokens": 96919756.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.3887341030407697, |
| "epoch": 7.248800661703887, |
| "grad_norm": 1.28125, |
| "learning_rate": 3.725006796400231e-06, |
| "loss": 0.385011625289917, |
| "mean_token_accuracy": 0.8824066145345568, |
| "num_tokens": 97630930.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.38799829850904644, |
| "epoch": 7.301736972704715, |
| "grad_norm": 1.3125, |
| "learning_rate": 3.5925589089542954e-06, |
| "loss": 0.38501543998718263, |
| "mean_token_accuracy": 0.8823068980127573, |
| "num_tokens": 98345419.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 0.3849033182952553, |
| "epoch": 7.354673283705542, |
| "grad_norm": 1.265625, |
| "learning_rate": 3.461991096121955e-06, |
| "loss": 0.3803849697113037, |
| "mean_token_accuracy": 0.8831438813358545, |
| "num_tokens": 99059531.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 0.38186661126092075, |
| "epoch": 7.4076095947063685, |
| "grad_norm": 1.203125, |
| "learning_rate": 3.3333416691770402e-06, |
| "loss": 0.37721364498138427, |
| "mean_token_accuracy": 0.8845844173803925, |
| "num_tokens": 99773516.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.38388882144354286, |
| "epoch": 7.460545905707196, |
| "grad_norm": 1.21875, |
| "learning_rate": 3.206648376499637e-06, |
| "loss": 0.377412223815918, |
| "mean_token_accuracy": 0.8840935161337257, |
| "num_tokens": 100494087.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.38188970838673414, |
| "epoch": 7.513482216708023, |
| "grad_norm": 1.2734375, |
| "learning_rate": 3.0819483924999183e-06, |
| "loss": 0.3805495262145996, |
| "mean_token_accuracy": 0.8833204042166471, |
| "num_tokens": 101214749.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 0.3855323393829167, |
| "epoch": 7.5664185277088505, |
| "grad_norm": 1.2421875, |
| "learning_rate": 2.959278306710409e-06, |
| "loss": 0.38122191429138186, |
| "mean_token_accuracy": 0.8832844633609056, |
| "num_tokens": 101926434.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 0.3883001868147403, |
| "epoch": 7.619354838709677, |
| "grad_norm": 1.2421875, |
| "learning_rate": 2.838674113049884e-06, |
| "loss": 0.3854327917098999, |
| "mean_token_accuracy": 0.8816704792901874, |
| "num_tokens": 102638578.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 0.39824472144246104, |
| "epoch": 7.672291149710505, |
| "grad_norm": 1.234375, |
| "learning_rate": 2.7201711992619873e-06, |
| "loss": 0.3931479215621948, |
| "mean_token_accuracy": 0.8806289885193109, |
| "num_tokens": 103358219.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.3936444675549865, |
| "epoch": 7.725227460711332, |
| "grad_norm": 1.2421875, |
| "learning_rate": 2.6038043365317643e-06, |
| "loss": 0.38866076469421384, |
| "mean_token_accuracy": 0.8816944660618902, |
| "num_tokens": 104064920.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 0.3860598919913173, |
| "epoch": 7.778163771712158, |
| "grad_norm": 1.21875, |
| "learning_rate": 2.489607669283088e-06, |
| "loss": 0.38236322402954104, |
| "mean_token_accuracy": 0.8831515850499272, |
| "num_tokens": 104774666.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.38925819350406526, |
| "epoch": 7.831100082712986, |
| "grad_norm": 1.2421875, |
| "learning_rate": 2.3776147051599518e-06, |
| "loss": 0.38701634407043456, |
| "mean_token_accuracy": 0.8821640037000179, |
| "num_tokens": 105479403.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.39431367358192804, |
| "epoch": 7.884036393713813, |
| "grad_norm": 1.1875, |
| "learning_rate": 2.2678583051946733e-06, |
| "loss": 0.39297478199005126, |
| "mean_token_accuracy": 0.8811518820002675, |
| "num_tokens": 106198229.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.3987340045627207, |
| "epoch": 7.93697270471464, |
| "grad_norm": 1.3359375, |
| "learning_rate": 2.1603706741657803e-06, |
| "loss": 0.39476895332336426, |
| "mean_token_accuracy": 0.880135708115995, |
| "num_tokens": 106903767.0, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.39117897003889085, |
| "epoch": 7.989909015715467, |
| "grad_norm": 1.1953125, |
| "learning_rate": 2.055183351148463e-06, |
| "loss": 0.3864224910736084, |
| "mean_token_accuracy": 0.8818761941045523, |
| "num_tokens": 107612975.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 0.38389241361655646, |
| "epoch": 8.042349048800661, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.9523272002603745e-06, |
| "loss": 0.3774052619934082, |
| "mean_token_accuracy": 0.8843309322742258, |
| "num_tokens": 108320998.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 0.38878993326798084, |
| "epoch": 8.095285359801489, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.8518324016054601e-06, |
| "loss": 0.3859711647033691, |
| "mean_token_accuracy": 0.8827047497034073, |
| "num_tokens": 109032408.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.3901982956100255, |
| "epoch": 8.148221670802316, |
| "grad_norm": 1.34375, |
| "learning_rate": 1.7537284424184852e-06, |
| "loss": 0.3861372470855713, |
| "mean_token_accuracy": 0.8820165496319532, |
| "num_tokens": 109748057.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 0.3954754684586078, |
| "epoch": 8.201157981803144, |
| "grad_norm": 1.25, |
| "learning_rate": 1.658044108412905e-06, |
| "loss": 0.39197049140930174, |
| "mean_token_accuracy": 0.8802052192389965, |
| "num_tokens": 110462975.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.3881538372952491, |
| "epoch": 8.25409429280397, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.564807475334509e-06, |
| "loss": 0.385103702545166, |
| "mean_token_accuracy": 0.8827815610915423, |
| "num_tokens": 111178903.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.39255551765672864, |
| "epoch": 8.307030603804797, |
| "grad_norm": 1.296875, |
| "learning_rate": 1.4740459007234498e-06, |
| "loss": 0.390416693687439, |
| "mean_token_accuracy": 0.8810080744326114, |
| "num_tokens": 111897536.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 0.38641950571909545, |
| "epoch": 8.359966914805625, |
| "grad_norm": 1.25, |
| "learning_rate": 1.3857860158869696e-06, |
| "loss": 0.3832695484161377, |
| "mean_token_accuracy": 0.8827085629105568, |
| "num_tokens": 112605102.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 0.39179602907970545, |
| "epoch": 8.412903225806451, |
| "grad_norm": 1.25, |
| "learning_rate": 1.3000537180852213e-06, |
| "loss": 0.3873727798461914, |
| "mean_token_accuracy": 0.8816867783665657, |
| "num_tokens": 113325128.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 0.39006129098124803, |
| "epoch": 8.465839536807279, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.2168741629324944e-06, |
| "loss": 0.38400955200195314, |
| "mean_token_accuracy": 0.8828409412875772, |
| "num_tokens": 114034422.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.37696139281615615, |
| "epoch": 8.518775847808106, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.1362717570160387e-06, |
| "loss": 0.3724513053894043, |
| "mean_token_accuracy": 0.8855425385758281, |
| "num_tokens": 114750388.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 0.3893483619205654, |
| "epoch": 8.571712158808934, |
| "grad_norm": 1.296875, |
| "learning_rate": 1.0582701507346582e-06, |
| "loss": 0.3848257064819336, |
| "mean_token_accuracy": 0.882491577975452, |
| "num_tokens": 115466046.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 0.3910596649162471, |
| "epoch": 8.62464846980976, |
| "grad_norm": 1.25, |
| "learning_rate": 9.828922313592227e-07, |
| "loss": 0.38617868423461915, |
| "mean_token_accuracy": 0.8822582213208079, |
| "num_tokens": 116185312.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.3851263083983213, |
| "epoch": 8.677584780810587, |
| "grad_norm": 1.2578125, |
| "learning_rate": 9.10160116317047e-07, |
| "loss": 0.38117475509643556, |
| "mean_token_accuracy": 0.8832471214234829, |
| "num_tokens": 116903441.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 0.39345112908631563, |
| "epoch": 8.730521091811415, |
| "grad_norm": 1.265625, |
| "learning_rate": 8.400951467022067e-07, |
| "loss": 0.39005289077758787, |
| "mean_token_accuracy": 0.881065203808248, |
| "num_tokens": 117609009.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.39129888704046606, |
| "epoch": 8.78345740281224, |
| "grad_norm": 1.234375, |
| "learning_rate": 7.727178810136094e-07, |
| "loss": 0.38665571212768557, |
| "mean_token_accuracy": 0.8822559352964163, |
| "num_tokens": 118312265.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 0.38305580699816344, |
| "epoch": 8.836393713813068, |
| "grad_norm": 1.234375, |
| "learning_rate": 7.080480891227126e-07, |
| "loss": 0.37691564559936525, |
| "mean_token_accuracy": 0.8844683155417442, |
| "num_tokens": 119025289.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 0.39032987314276396, |
| "epoch": 8.889330024813896, |
| "grad_norm": 1.2265625, |
| "learning_rate": 6.461047464726433e-07, |
| "loss": 0.38719003200531005, |
| "mean_token_accuracy": 0.8823133103549481, |
| "num_tokens": 119739051.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 0.3877064341213554, |
| "epoch": 8.942266335814724, |
| "grad_norm": 1.3046875, |
| "learning_rate": 5.869060285104089e-07, |
| "loss": 0.384942626953125, |
| "mean_token_accuracy": 0.8828741613775491, |
| "num_tokens": 120447293.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.3802658010274172, |
| "epoch": 8.99520264681555, |
| "grad_norm": 1.1640625, |
| "learning_rate": 5.304693053538434e-07, |
| "loss": 0.37456960678100587, |
| "mean_token_accuracy": 0.8852874081581831, |
| "num_tokens": 121152193.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.3825313971621757, |
| "epoch": 9.047642679900745, |
| "grad_norm": 1.2265625, |
| "learning_rate": 4.768111366948736e-07, |
| "loss": 0.3785205125808716, |
| "mean_token_accuracy": 0.8844638754516746, |
| "num_tokens": 121857419.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 0.3895401722751558, |
| "epoch": 9.100578990901571, |
| "grad_norm": 1.2734375, |
| "learning_rate": 4.2594726694055397e-07, |
| "loss": 0.38504986763000487, |
| "mean_token_accuracy": 0.882383463345468, |
| "num_tokens": 122583636.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.3900445295963436, |
| "epoch": 9.153515301902399, |
| "grad_norm": 1.2265625, |
| "learning_rate": 3.778926205933342e-07, |
| "loss": 0.38513941764831544, |
| "mean_token_accuracy": 0.8823683267459274, |
| "num_tokens": 123297381.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.3941331796348095, |
| "epoch": 9.206451612903226, |
| "grad_norm": 1.234375, |
| "learning_rate": 3.3266129787191015e-07, |
| "loss": 0.3919462442398071, |
| "mean_token_accuracy": 0.880586769618094, |
| "num_tokens": 124011276.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.3884563890751451, |
| "epoch": 9.259387923904052, |
| "grad_norm": 1.2578125, |
| "learning_rate": 2.9026657057392715e-07, |
| "loss": 0.38395793437957765, |
| "mean_token_accuracy": 0.882823952101171, |
| "num_tokens": 124728740.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.3854251720942557, |
| "epoch": 9.31232423490488, |
| "grad_norm": 1.2890625, |
| "learning_rate": 2.507208781817638e-07, |
| "loss": 0.38173847198486327, |
| "mean_token_accuracy": 0.8830624982714653, |
| "num_tokens": 125444088.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.38771451893262565, |
| "epoch": 9.365260545905707, |
| "grad_norm": 1.265625, |
| "learning_rate": 2.1403582421253398e-07, |
| "loss": 0.3813947200775146, |
| "mean_token_accuracy": 0.8836282709613442, |
| "num_tokens": 126153025.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.38431605296209453, |
| "epoch": 9.418196856906535, |
| "grad_norm": 1.2109375, |
| "learning_rate": 1.8022217281336908e-07, |
| "loss": 0.3796398639678955, |
| "mean_token_accuracy": 0.8836826490238309, |
| "num_tokens": 126874049.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 0.38797196205705403, |
| "epoch": 9.471133167907361, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.4928984560301006e-07, |
| "loss": 0.38299384117126467, |
| "mean_token_accuracy": 0.8831497086212039, |
| "num_tokens": 127587154.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.39351940583437683, |
| "epoch": 9.524069478908189, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.212479187605897e-07, |
| "loss": 0.38923139572143556, |
| "mean_token_accuracy": 0.881783135049045, |
| "num_tokens": 128290741.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.39206409207545223, |
| "epoch": 9.577005789909016, |
| "grad_norm": 1.28125, |
| "learning_rate": 9.610462036249823e-08, |
| "loss": 0.388545823097229, |
| "mean_token_accuracy": 0.8814939023926854, |
| "num_tokens": 129010702.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.38616840010508896, |
| "epoch": 9.629942100909842, |
| "grad_norm": 1.2265625, |
| "learning_rate": 7.386732796810014e-08, |
| "loss": 0.3817445278167725, |
| "mean_token_accuracy": 0.8830347690731287, |
| "num_tokens": 129724831.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 0.3846033324021846, |
| "epoch": 9.68287841191067, |
| "grad_norm": 1.234375, |
| "learning_rate": 5.454256645500477e-08, |
| "loss": 0.37910990715026854, |
| "mean_token_accuracy": 0.8839818915352226, |
| "num_tokens": 130432884.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.39172688918188214, |
| "epoch": 9.735814722911497, |
| "grad_norm": 1.2734375, |
| "learning_rate": 3.8136006104533276e-08, |
| "loss": 0.38709568977355957, |
| "mean_token_accuracy": 0.882011273689568, |
| "num_tokens": 131154186.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 0.38724804315716027, |
| "epoch": 9.788751033912325, |
| "grad_norm": 1.265625, |
| "learning_rate": 2.4652460937941714e-08, |
| "loss": 0.3846461296081543, |
| "mean_token_accuracy": 0.8825461909174919, |
| "num_tokens": 131867648.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.3888995597604662, |
| "epoch": 9.84168734491315, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.4095887303888778e-08, |
| "loss": 0.38567285537719725, |
| "mean_token_accuracy": 0.88200354706496, |
| "num_tokens": 132562489.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 0.38812326178886, |
| "epoch": 9.894623655913978, |
| "grad_norm": 1.265625, |
| "learning_rate": 6.469382717563255e-09, |
| "loss": 0.38445029258728025, |
| "mean_token_accuracy": 0.8823262698948383, |
| "num_tokens": 133272125.0, |
| "step": 1870 |
| }, |
| { |
| "entropy": 0.3832914155907929, |
| "epoch": 9.947559966914806, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.775184951806619e-09, |
| "loss": 0.37877960205078126, |
| "mean_token_accuracy": 0.8840421948581934, |
| "num_tokens": 133981204.0, |
| "step": 1880 |
| }, |
| { |
| "entropy": 0.3908322428115159, |
| "epoch": 10.0, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.4671380509367893e-11, |
| "loss": 0.3877368927001953, |
| "mean_token_accuracy": 0.8816912356611306, |
| "num_tokens": 134689470.0, |
| "step": 1890 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1890, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.6360874475383685e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
