| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.0, |
| "eval_steps": 500, |
| "global_step": 1890, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 2.517383523657918, |
| "epoch": 0.05293631100082713, |
| "grad_norm": 21.875, |
| "learning_rate": 3.2142857142857147e-06, |
| "loss": 2.330343818664551, |
| "mean_token_accuracy": 0.5431291756220162, |
| "num_tokens": 710093.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.7889304729178548, |
| "epoch": 0.10587262200165426, |
| "grad_norm": 6.0, |
| "learning_rate": 6.785714285714287e-06, |
| "loss": 1.719919204711914, |
| "mean_token_accuracy": 0.6246315483935178, |
| "num_tokens": 1429271.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.1666983438655734, |
| "epoch": 0.1588089330024814, |
| "grad_norm": 2.71875, |
| "learning_rate": 1.0357142857142859e-05, |
| "loss": 1.2366750717163086, |
| "mean_token_accuracy": 0.7033325288444757, |
| "num_tokens": 2149735.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.9782258225604892, |
| "epoch": 0.21174524400330852, |
| "grad_norm": 1.78125, |
| "learning_rate": 1.3928571428571429e-05, |
| "loss": 0.9922978401184082, |
| "mean_token_accuracy": 0.7494580747559667, |
| "num_tokens": 2868867.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.8958244953304529, |
| "epoch": 0.26468155500413565, |
| "grad_norm": 1.71875, |
| "learning_rate": 1.7500000000000002e-05, |
| "loss": 0.9041233062744141, |
| "mean_token_accuracy": 0.767370667308569, |
| "num_tokens": 3582476.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.8279623622074723, |
| "epoch": 0.3176178660049628, |
| "grad_norm": 1.5546875, |
| "learning_rate": 1.999986795783372e-05, |
| "loss": 0.8340991020202637, |
| "mean_token_accuracy": 0.782540894113481, |
| "num_tokens": 4287689.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.8150905193760991, |
| "epoch": 0.3705541770057899, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.9997520638548385e-05, |
| "loss": 0.8254405975341796, |
| "mean_token_accuracy": 0.783768461458385, |
| "num_tokens": 5011191.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.7939997553825379, |
| "epoch": 0.42349048800661704, |
| "grad_norm": 1.4140625, |
| "learning_rate": 1.9992239841688418e-05, |
| "loss": 0.7975911140441895, |
| "mean_token_accuracy": 0.7906581481918693, |
| "num_tokens": 5714842.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.7681451645679772, |
| "epoch": 0.47642679900744417, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.9984027116747914e-05, |
| "loss": 0.7691312789916992, |
| "mean_token_accuracy": 0.7958846906200051, |
| "num_tokens": 6424614.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.7580096147023141, |
| "epoch": 0.5293631100082713, |
| "grad_norm": 1.3671875, |
| "learning_rate": 1.997288487350878e-05, |
| "loss": 0.7647263526916503, |
| "mean_token_accuracy": 0.7970432203263045, |
| "num_tokens": 7139778.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.7555975493974983, |
| "epoch": 0.5822994210090985, |
| "grad_norm": 1.8046875, |
| "learning_rate": 1.9958816381333684e-05, |
| "loss": 0.7625592708587646, |
| "mean_token_accuracy": 0.7988613218069076, |
| "num_tokens": 7852540.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.7381295645609498, |
| "epoch": 0.6352357320099256, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.994182576820673e-05, |
| "loss": 0.7393324851989747, |
| "mean_token_accuracy": 0.8023298803716898, |
| "num_tokens": 8568056.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.7293703068047762, |
| "epoch": 0.6881720430107527, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.9921918019522238e-05, |
| "loss": 0.7372323036193847, |
| "mean_token_accuracy": 0.8033495251089334, |
| "num_tokens": 9276742.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.727802424877882, |
| "epoch": 0.7411083540115798, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.9899098976621922e-05, |
| "loss": 0.7326857089996338, |
| "mean_token_accuracy": 0.8046217462047934, |
| "num_tokens": 9990163.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.7251318506896496, |
| "epoch": 0.794044665012407, |
| "grad_norm": 1.3984375, |
| "learning_rate": 1.9873375335080915e-05, |
| "loss": 0.7273187160491943, |
| "mean_token_accuracy": 0.8048763429746032, |
| "num_tokens": 10710006.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.7191944925114513, |
| "epoch": 0.8469809760132341, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.9844754642743164e-05, |
| "loss": 0.7239957809448242, |
| "mean_token_accuracy": 0.8055033938959241, |
| "num_tokens": 11416822.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.7240813641808928, |
| "epoch": 0.8999172870140613, |
| "grad_norm": 1.2578125, |
| "learning_rate": 1.981324529750672e-05, |
| "loss": 0.7269858837127685, |
| "mean_token_accuracy": 0.8054354954510927, |
| "num_tokens": 12123118.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.715485874377191, |
| "epoch": 0.9528535980148883, |
| "grad_norm": 1.359375, |
| "learning_rate": 1.9778856544859647e-05, |
| "loss": 0.7194882392883301, |
| "mean_token_accuracy": 0.8073206223547459, |
| "num_tokens": 12831058.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.7016689017744094, |
| "epoch": 1.0052936311000826, |
| "grad_norm": 1.2578125, |
| "learning_rate": 1.9741598475167173e-05, |
| "loss": 0.6987634658813476, |
| "mean_token_accuracy": 0.8111047323569891, |
| "num_tokens": 13539870.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.6788103650324047, |
| "epoch": 1.0582299421009098, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.9701482020711004e-05, |
| "loss": 0.6810420989990235, |
| "mean_token_accuracy": 0.8135505892336369, |
| "num_tokens": 14256699.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.6636494144797325, |
| "epoch": 1.111166253101737, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.9658518952481543e-05, |
| "loss": 0.667720365524292, |
| "mean_token_accuracy": 0.817850429750979, |
| "num_tokens": 14977492.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.6727796928957105, |
| "epoch": 1.1641025641025642, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.9612721876724054e-05, |
| "loss": 0.6756388664245605, |
| "mean_token_accuracy": 0.8159420626237989, |
| "num_tokens": 15693723.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.6585825039073825, |
| "epoch": 1.2170388751033911, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.9564104231239726e-05, |
| "loss": 0.6569587707519531, |
| "mean_token_accuracy": 0.818596726283431, |
| "num_tokens": 16404449.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.658372915815562, |
| "epoch": 1.2699751861042183, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.9512680281442745e-05, |
| "loss": 0.6602190494537353, |
| "mean_token_accuracy": 0.8187425743788481, |
| "num_tokens": 17118450.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.664527343865484, |
| "epoch": 1.3229114971050455, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.9458465116174534e-05, |
| "loss": 0.6621711730957032, |
| "mean_token_accuracy": 0.8184419617056846, |
| "num_tokens": 17826861.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.66327049927786, |
| "epoch": 1.3758478081058727, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.940147464327637e-05, |
| "loss": 0.6667583465576172, |
| "mean_token_accuracy": 0.8170910779386759, |
| "num_tokens": 18545340.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.6499099671840668, |
| "epoch": 1.4287841191067, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.9341725584921712e-05, |
| "loss": 0.6517716884613037, |
| "mean_token_accuracy": 0.8211767397820949, |
| "num_tokens": 19262118.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.652203946840018, |
| "epoch": 1.4817204301075269, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.9279235472709554e-05, |
| "loss": 0.6548329830169678, |
| "mean_token_accuracy": 0.8199291430413723, |
| "num_tokens": 19968479.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.6609947566874326, |
| "epoch": 1.534656741108354, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.92140226425203e-05, |
| "loss": 0.6620079040527344, |
| "mean_token_accuracy": 0.8178581858053804, |
| "num_tokens": 20668540.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.6583634492009878, |
| "epoch": 1.587593052109181, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.914610622913566e-05, |
| "loss": 0.6613290786743165, |
| "mean_token_accuracy": 0.8191439069807529, |
| "num_tokens": 21377905.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.6483378401957453, |
| "epoch": 1.6405293631100082, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.9075506160624068e-05, |
| "loss": 0.6520229339599609, |
| "mean_token_accuracy": 0.8208056768402457, |
| "num_tokens": 22086263.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.6439661994576454, |
| "epoch": 1.6934656741108354, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.900224315249343e-05, |
| "loss": 0.6434724807739258, |
| "mean_token_accuracy": 0.8223148621618748, |
| "num_tokens": 22790833.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.6394708540290595, |
| "epoch": 1.7464019851116626, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.8926338701612738e-05, |
| "loss": 0.6371979236602783, |
| "mean_token_accuracy": 0.8228612139821052, |
| "num_tokens": 23503166.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.648493236489594, |
| "epoch": 1.7993382961124897, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.884781507990445e-05, |
| "loss": 0.650775146484375, |
| "mean_token_accuracy": 0.8212731294333935, |
| "num_tokens": 24223112.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.6460507491603493, |
| "epoch": 1.852274607113317, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.8766695327809463e-05, |
| "loss": 0.6457096576690674, |
| "mean_token_accuracy": 0.822027931548655, |
| "num_tokens": 24936212.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.6296045129187405, |
| "epoch": 1.905210918114144, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.8683003247526567e-05, |
| "loss": 0.6345595359802246, |
| "mean_token_accuracy": 0.8248383782804012, |
| "num_tokens": 25662368.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.642846689838916, |
| "epoch": 1.958147229114971, |
| "grad_norm": 1.25, |
| "learning_rate": 1.8596763396028384e-05, |
| "loss": 0.6466265201568604, |
| "mean_token_accuracy": 0.822427730448544, |
| "num_tokens": 26371713.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.6445435251723328, |
| "epoch": 2.0105872622001653, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.8508001077855876e-05, |
| "loss": 0.640605115890503, |
| "mean_token_accuracy": 0.8237532322715133, |
| "num_tokens": 27082791.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.5935637545771897, |
| "epoch": 2.0635235732009924, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.8416742337693436e-05, |
| "loss": 0.5961747646331788, |
| "mean_token_accuracy": 0.832476151548326, |
| "num_tokens": 27804616.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.6074226231314241, |
| "epoch": 2.1164598842018196, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.8323013952726877e-05, |
| "loss": 0.6060761451721192, |
| "mean_token_accuracy": 0.8300150576978922, |
| "num_tokens": 28510447.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.6045498767867684, |
| "epoch": 2.169396195202647, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.8226843424786434e-05, |
| "loss": 0.6060684204101563, |
| "mean_token_accuracy": 0.8299270933493972, |
| "num_tokens": 29231259.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.5919496919959784, |
| "epoch": 2.222332506203474, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.8128258972277176e-05, |
| "loss": 0.5906681060791016, |
| "mean_token_accuracy": 0.8336266664788127, |
| "num_tokens": 29952760.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.5975334191694855, |
| "epoch": 2.275268817204301, |
| "grad_norm": 1.140625, |
| "learning_rate": 1.8027289521899143e-05, |
| "loss": 0.6016942501068115, |
| "mean_token_accuracy": 0.8313274193555117, |
| "num_tokens": 30665899.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.6087098903022706, |
| "epoch": 2.3282051282051284, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.7923964700159658e-05, |
| "loss": 0.6066548347473144, |
| "mean_token_accuracy": 0.8301778499037027, |
| "num_tokens": 31372484.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.6126051696948707, |
| "epoch": 2.3811414392059556, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.78183148246803e-05, |
| "loss": 0.614737606048584, |
| "mean_token_accuracy": 0.8285853797569871, |
| "num_tokens": 32096632.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.5990028641186654, |
| "epoch": 2.4340777502067823, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.77103708953011e-05, |
| "loss": 0.6006610870361329, |
| "mean_token_accuracy": 0.8312915643677116, |
| "num_tokens": 32804805.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.607753461226821, |
| "epoch": 2.4870140612076095, |
| "grad_norm": 1.2109375, |
| "learning_rate": 1.7600164584984548e-05, |
| "loss": 0.6072165966033936, |
| "mean_token_accuracy": 0.8301842240616679, |
| "num_tokens": 33508738.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.595335545297712, |
| "epoch": 2.5399503722084367, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.7487728230522085e-05, |
| "loss": 0.5950441360473633, |
| "mean_token_accuracy": 0.8320430673658847, |
| "num_tokens": 34225982.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.6014379073865712, |
| "epoch": 2.592886683209264, |
| "grad_norm": 1.1015625, |
| "learning_rate": 1.737309482304585e-05, |
| "loss": 0.602420711517334, |
| "mean_token_accuracy": 0.8312459668144584, |
| "num_tokens": 34943657.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.6020066059194505, |
| "epoch": 2.645822994210091, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.7256297998348388e-05, |
| "loss": 0.601630687713623, |
| "mean_token_accuracy": 0.8309382183477283, |
| "num_tokens": 35644283.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.5982570440508426, |
| "epoch": 2.6987593052109182, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.7137372027013212e-05, |
| "loss": 0.5992427825927734, |
| "mean_token_accuracy": 0.8313212908804417, |
| "num_tokens": 36350379.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.5970497015863657, |
| "epoch": 2.7516956162117454, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.7016351804359134e-05, |
| "loss": 0.5992032527923584, |
| "mean_token_accuracy": 0.8327719148248434, |
| "num_tokens": 37060303.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.6053770511411131, |
| "epoch": 2.804631927212572, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.6893272840201243e-05, |
| "loss": 0.6047993659973144, |
| "mean_token_accuracy": 0.8311333157122135, |
| "num_tokens": 37772753.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.5910255504772067, |
| "epoch": 2.8575682382134, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.6768171248431603e-05, |
| "loss": 0.5963948249816895, |
| "mean_token_accuracy": 0.832998582161963, |
| "num_tokens": 38489638.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.5908184374682606, |
| "epoch": 2.9105045492142265, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.66410837364227e-05, |
| "loss": 0.5898349285125732, |
| "mean_token_accuracy": 0.8335007596760988, |
| "num_tokens": 39198407.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.598981601651758, |
| "epoch": 2.9634408602150537, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.651204759425674e-05, |
| "loss": 0.5970761775970459, |
| "mean_token_accuracy": 0.8321403747424483, |
| "num_tokens": 39917572.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.593930964872288, |
| "epoch": 3.015880893300248, |
| "grad_norm": 1.125, |
| "learning_rate": 1.6381100683783984e-05, |
| "loss": 0.5896023273468017, |
| "mean_token_accuracy": 0.8338961172555147, |
| "num_tokens": 40623043.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 0.5697517069987953, |
| "epoch": 3.0688172043010753, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.6248281427513293e-05, |
| "loss": 0.5749466896057129, |
| "mean_token_accuracy": 0.8362925421446562, |
| "num_tokens": 41339395.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.5773605431430042, |
| "epoch": 3.1217535153019025, |
| "grad_norm": 1.125, |
| "learning_rate": 1.6113628797338182e-05, |
| "loss": 0.5751814842224121, |
| "mean_token_accuracy": 0.8367513783276082, |
| "num_tokens": 42048417.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.5606058123521507, |
| "epoch": 3.1746898263027297, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.597718230310167e-05, |
| "loss": 0.5579678535461425, |
| "mean_token_accuracy": 0.8400175305083394, |
| "num_tokens": 42764993.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.5559274241328239, |
| "epoch": 3.227626137303557, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.5838981981003273e-05, |
| "loss": 0.5564671039581299, |
| "mean_token_accuracy": 0.8413354951888323, |
| "num_tokens": 43484113.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 0.5620169792324304, |
| "epoch": 3.2805624483043836, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.5699068381851576e-05, |
| "loss": 0.5637299537658691, |
| "mean_token_accuracy": 0.8396881816908717, |
| "num_tokens": 44190080.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 0.5724300829693675, |
| "epoch": 3.333498759305211, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.5557482559165777e-05, |
| "loss": 0.5707749366760254, |
| "mean_token_accuracy": 0.8376087129116059, |
| "num_tokens": 44901717.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.5594174157828092, |
| "epoch": 3.386435070306038, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.541426605712975e-05, |
| "loss": 0.5598674297332764, |
| "mean_token_accuracy": 0.8401424733921885, |
| "num_tokens": 45611553.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.5719467986375093, |
| "epoch": 3.439371381306865, |
| "grad_norm": 1.125, |
| "learning_rate": 1.526946089840214e-05, |
| "loss": 0.5716081619262695, |
| "mean_token_accuracy": 0.8369537137448788, |
| "num_tokens": 46319422.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.5649781067855656, |
| "epoch": 3.4923076923076923, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.5123109571786034e-05, |
| "loss": 0.5649839401245117, |
| "mean_token_accuracy": 0.8399968167766929, |
| "num_tokens": 47040011.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.5679076578468084, |
| "epoch": 3.5452440033085195, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.4975255019761896e-05, |
| "loss": 0.5683752536773682, |
| "mean_token_accuracy": 0.8376537404954434, |
| "num_tokens": 47747908.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.5690185854211449, |
| "epoch": 3.5981803143093467, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.4825940625887343e-05, |
| "loss": 0.568476676940918, |
| "mean_token_accuracy": 0.8382133601233364, |
| "num_tokens": 48465834.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.5643869173713029, |
| "epoch": 3.6511166253101734, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.467521020206753e-05, |
| "loss": 0.5652931690216064, |
| "mean_token_accuracy": 0.8385411059483886, |
| "num_tokens": 49175962.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.5684676690027117, |
| "epoch": 3.704052936311001, |
| "grad_norm": 1.125, |
| "learning_rate": 1.4523107975699833e-05, |
| "loss": 0.565020751953125, |
| "mean_token_accuracy": 0.8384542506188154, |
| "num_tokens": 49891839.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.5727035844698548, |
| "epoch": 3.756989247311828, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.4369678576696633e-05, |
| "loss": 0.5756559371948242, |
| "mean_token_accuracy": 0.8364196993410588, |
| "num_tokens": 50614080.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.5612221390008927, |
| "epoch": 3.809925558312655, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.4214967024389978e-05, |
| "loss": 0.560767412185669, |
| "mean_token_accuracy": 0.8398452011868358, |
| "num_tokens": 51323270.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.5644309324212372, |
| "epoch": 3.862861869313482, |
| "grad_norm": 1.109375, |
| "learning_rate": 1.4059018714321986e-05, |
| "loss": 0.5635293006896973, |
| "mean_token_accuracy": 0.83980608060956, |
| "num_tokens": 52026181.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.5582810510881245, |
| "epoch": 3.9157981803143094, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.390187940492486e-05, |
| "loss": 0.5593689918518067, |
| "mean_token_accuracy": 0.8408091176301241, |
| "num_tokens": 52734953.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.5543095638975502, |
| "epoch": 3.9687344913151366, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.3743595204094442e-05, |
| "loss": 0.5532174110412598, |
| "mean_token_accuracy": 0.8413080271333456, |
| "num_tokens": 53453384.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.5514063389519411, |
| "epoch": 4.0211745244003305, |
| "grad_norm": 1.125, |
| "learning_rate": 1.3584212555661187e-05, |
| "loss": 0.546474552154541, |
| "mean_token_accuracy": 0.8432126494612228, |
| "num_tokens": 54163412.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.538205324113369, |
| "epoch": 4.074110835401158, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.3423778225762609e-05, |
| "loss": 0.538818073272705, |
| "mean_token_accuracy": 0.8444410042837263, |
| "num_tokens": 54870856.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.5450183315202594, |
| "epoch": 4.127047146401985, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.3262339289121129e-05, |
| "loss": 0.5455585479736328, |
| "mean_token_accuracy": 0.8425258509814739, |
| "num_tokens": 55582308.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.5478194649331272, |
| "epoch": 4.1799834574028125, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.3099943115231386e-05, |
| "loss": 0.5473073959350586, |
| "mean_token_accuracy": 0.8427304636687041, |
| "num_tokens": 56302219.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.548308833874762, |
| "epoch": 4.232919768403639, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.2936637354461064e-05, |
| "loss": 0.5452468872070313, |
| "mean_token_accuracy": 0.8430783998221159, |
| "num_tokens": 57015624.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.5472051261924207, |
| "epoch": 4.285856079404467, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.2772469924069285e-05, |
| "loss": 0.5482036590576171, |
| "mean_token_accuracy": 0.8431986581534148, |
| "num_tokens": 57728689.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.5367700139060616, |
| "epoch": 4.338792390405294, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.2607488994146703e-05, |
| "loss": 0.5386350631713868, |
| "mean_token_accuracy": 0.8448256650939584, |
| "num_tokens": 58440887.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.5441102403216064, |
| "epoch": 4.39172870140612, |
| "grad_norm": 1.140625, |
| "learning_rate": 1.2441742973481432e-05, |
| "loss": 0.5408960819244385, |
| "mean_token_accuracy": 0.8442874392494559, |
| "num_tokens": 59161499.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.5355542128905654, |
| "epoch": 4.444665012406948, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.2275280495354889e-05, |
| "loss": 0.534831190109253, |
| "mean_token_accuracy": 0.8461610395461321, |
| "num_tokens": 59872509.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.5415700157172978, |
| "epoch": 4.497601323407775, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.21081504032718e-05, |
| "loss": 0.539634895324707, |
| "mean_token_accuracy": 0.8452536968514324, |
| "num_tokens": 60593833.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.536719624325633, |
| "epoch": 4.550537634408602, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.1940401736628524e-05, |
| "loss": 0.535213565826416, |
| "mean_token_accuracy": 0.8448078541085124, |
| "num_tokens": 61303595.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.5386149102821947, |
| "epoch": 4.603473945409429, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.1772083716323878e-05, |
| "loss": 0.5384332180023194, |
| "mean_token_accuracy": 0.844671962223947, |
| "num_tokens": 62014565.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.5413594543933868, |
| "epoch": 4.656410256410257, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.1603245730316724e-05, |
| "loss": 0.5381780624389648, |
| "mean_token_accuracy": 0.843700847402215, |
| "num_tokens": 62723366.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.5437364445999264, |
| "epoch": 4.7093465674110835, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.1433937319134512e-05, |
| "loss": 0.541628646850586, |
| "mean_token_accuracy": 0.8439845886081457, |
| "num_tokens": 63432928.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.5380360207520425, |
| "epoch": 4.762282878411911, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.1264208161337091e-05, |
| "loss": 0.5398398399353027, |
| "mean_token_accuracy": 0.8448185469955206, |
| "num_tokens": 64137723.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.5411873222328722, |
| "epoch": 4.815219189412738, |
| "grad_norm": 1.125, |
| "learning_rate": 1.1094108058940005e-05, |
| "loss": 0.5392769813537598, |
| "mean_token_accuracy": 0.8441998787224293, |
| "num_tokens": 64851083.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.5336352121084929, |
| "epoch": 4.868155500413565, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.0923686922801537e-05, |
| "loss": 0.5329482078552246, |
| "mean_token_accuracy": 0.8462087785825133, |
| "num_tokens": 65572798.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.5287617506459356, |
| "epoch": 4.921091811414392, |
| "grad_norm": 1.125, |
| "learning_rate": 1.0752994757977862e-05, |
| "loss": 0.5266700744628906, |
| "mean_token_accuracy": 0.8476861402392387, |
| "num_tokens": 66285398.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.53608170254156, |
| "epoch": 4.974028122415219, |
| "grad_norm": 1.109375, |
| "learning_rate": 1.058208164905052e-05, |
| "loss": 0.5371575355529785, |
| "mean_token_accuracy": 0.8453869987279177, |
| "num_tokens": 66999572.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.5327976091238978, |
| "epoch": 5.026468155500414, |
| "grad_norm": 1.1171875, |
| "learning_rate": 1.041099774543059e-05, |
| "loss": 0.5274302005767822, |
| "mean_token_accuracy": 0.8470476427664893, |
| "num_tokens": 67710110.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.5196379259228706, |
| "epoch": 5.079404466501241, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.023979324664382e-05, |
| "loss": 0.5175643920898437, |
| "mean_token_accuracy": 0.8488413186743855, |
| "num_tokens": 68425350.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.517873155605048, |
| "epoch": 5.132340777502068, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.0068518387601038e-05, |
| "loss": 0.5160090446472168, |
| "mean_token_accuracy": 0.8491935424506665, |
| "num_tokens": 69142159.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.5161537874490023, |
| "epoch": 5.185277088502895, |
| "grad_norm": 1.171875, |
| "learning_rate": 9.897223423858226e-06, |
| "loss": 0.5144171714782715, |
| "mean_token_accuracy": 0.8498559452593326, |
| "num_tokens": 69854650.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.5172417410649359, |
| "epoch": 5.238213399503722, |
| "grad_norm": 1.1328125, |
| "learning_rate": 9.725958616870495e-06, |
| "loss": 0.5163768768310547, |
| "mean_token_accuracy": 0.8498215934261679, |
| "num_tokens": 70570857.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.5334854922257364, |
| "epoch": 5.291149710504549, |
| "grad_norm": 1.2578125, |
| "learning_rate": 9.554774219244344e-06, |
| "loss": 0.531788444519043, |
| "mean_token_accuracy": 0.845740943774581, |
| "num_tokens": 71287442.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.5226979675702751, |
| "epoch": 5.344086021505376, |
| "grad_norm": 1.09375, |
| "learning_rate": 9.38372045999252e-06, |
| "loss": 0.5185129165649414, |
| "mean_token_accuracy": 0.8491301748901605, |
| "num_tokens": 72007448.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.5161179441027344, |
| "epoch": 5.397022332506204, |
| "grad_norm": 1.1875, |
| "learning_rate": 9.212847529795784e-06, |
| "loss": 0.5171782016754151, |
| "mean_token_accuracy": 0.8490116039291025, |
| "num_tokens": 72715352.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.527431449200958, |
| "epoch": 5.44995864350703, |
| "grad_norm": 1.1328125, |
| "learning_rate": 9.042205566275952e-06, |
| "loss": 0.5250188827514648, |
| "mean_token_accuracy": 0.8475877868011594, |
| "num_tokens": 73429166.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.5290888456627727, |
| "epoch": 5.502894954507858, |
| "grad_norm": 1.1328125, |
| "learning_rate": 8.871844639284479e-06, |
| "loss": 0.5255687713623047, |
| "mean_token_accuracy": 0.8473750628530979, |
| "num_tokens": 74133406.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.5290718507021666, |
| "epoch": 5.555831265508685, |
| "grad_norm": 1.1171875, |
| "learning_rate": 8.701814736210933e-06, |
| "loss": 0.5295787811279297, |
| "mean_token_accuracy": 0.846999473683536, |
| "num_tokens": 74836672.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.5218891215510666, |
| "epoch": 5.608767576509512, |
| "grad_norm": 1.1796875, |
| "learning_rate": 8.532165747315686e-06, |
| "loss": 0.5223793983459473, |
| "mean_token_accuracy": 0.8479848142713309, |
| "num_tokens": 75557055.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.5259200980886817, |
| "epoch": 5.661703887510339, |
| "grad_norm": 1.265625, |
| "learning_rate": 8.362947451091072e-06, |
| "loss": 0.5238901138305664, |
| "mean_token_accuracy": 0.8482968054711819, |
| "num_tokens": 76272061.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.5228938857093454, |
| "epoch": 5.714640198511166, |
| "grad_norm": 1.1484375, |
| "learning_rate": 8.194209499655364e-06, |
| "loss": 0.5253058433532715, |
| "mean_token_accuracy": 0.8477638551965356, |
| "num_tokens": 76989852.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 0.5306269845925271, |
| "epoch": 5.7675765095119935, |
| "grad_norm": 1.2109375, |
| "learning_rate": 8.026001404183831e-06, |
| "loss": 0.5262615203857421, |
| "mean_token_accuracy": 0.8476999465376138, |
| "num_tokens": 77687094.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.5304377974942327, |
| "epoch": 5.82051282051282, |
| "grad_norm": 1.15625, |
| "learning_rate": 7.85837252038112e-06, |
| "loss": 0.5329948425292969, |
| "mean_token_accuracy": 0.8457729490473866, |
| "num_tokens": 78397944.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.5324696778319776, |
| "epoch": 5.873449131513648, |
| "grad_norm": 1.1640625, |
| "learning_rate": 7.69137203399931e-06, |
| "loss": 0.5273681640625, |
| "mean_token_accuracy": 0.8466610265895724, |
| "num_tokens": 79107666.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.5180164442397655, |
| "epoch": 5.926385442514475, |
| "grad_norm": 1.1328125, |
| "learning_rate": 7.525048946405791e-06, |
| "loss": 0.518718147277832, |
| "mean_token_accuracy": 0.8502681368961931, |
| "num_tokens": 79824629.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 0.5257153943181038, |
| "epoch": 5.979321753515302, |
| "grad_norm": 1.203125, |
| "learning_rate": 7.359452060205245e-06, |
| "loss": 0.5247107028961182, |
| "mean_token_accuracy": 0.8476573526859283, |
| "num_tokens": 80532805.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 0.5179383000740869, |
| "epoch": 6.031761786600496, |
| "grad_norm": 1.234375, |
| "learning_rate": 7.194629964919989e-06, |
| "loss": 0.5136065006256103, |
| "mean_token_accuracy": 0.8496653262749079, |
| "num_tokens": 81243632.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.5040348807349802, |
| "epoch": 6.084698097601324, |
| "grad_norm": 1.1484375, |
| "learning_rate": 7.030631022732781e-06, |
| "loss": 0.5017824172973633, |
| "mean_token_accuracy": 0.8528982033953071, |
| "num_tokens": 81955824.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.5168998019769788, |
| "epoch": 6.137634408602151, |
| "grad_norm": 1.2265625, |
| "learning_rate": 6.867503354296392e-06, |
| "loss": 0.5152142524719239, |
| "mean_token_accuracy": 0.8497169893234968, |
| "num_tokens": 82672365.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 0.5297345926985144, |
| "epoch": 6.190570719602977, |
| "grad_norm": 1.21875, |
| "learning_rate": 6.705294824614004e-06, |
| "loss": 0.5261876106262207, |
| "mean_token_accuracy": 0.8466440049931407, |
| "num_tokens": 83385723.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.5218852560035885, |
| "epoch": 6.243507030603805, |
| "grad_norm": 1.1640625, |
| "learning_rate": 6.54405302899463e-06, |
| "loss": 0.5192302227020263, |
| "mean_token_accuracy": 0.8485880514606834, |
| "num_tokens": 84097235.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.5227232126519084, |
| "epoch": 6.296443341604632, |
| "grad_norm": 1.1328125, |
| "learning_rate": 6.383825279087699e-06, |
| "loss": 0.5190953731536865, |
| "mean_token_accuracy": 0.8483411800116301, |
| "num_tokens": 84809815.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.514331001508981, |
| "epoch": 6.349379652605459, |
| "grad_norm": 1.1875, |
| "learning_rate": 6.2246585890008245e-06, |
| "loss": 0.5148365497589111, |
| "mean_token_accuracy": 0.8502315638586879, |
| "num_tokens": 85522179.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.5077279037795961, |
| "epoch": 6.402315963606286, |
| "grad_norm": 1.2109375, |
| "learning_rate": 6.06659966150491e-06, |
| "loss": 0.5086464405059814, |
| "mean_token_accuracy": 0.8512278325855732, |
| "num_tokens": 86246555.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.514440215844661, |
| "epoch": 6.455252274607114, |
| "grad_norm": 1.171875, |
| "learning_rate": 5.909694874330601e-06, |
| "loss": 0.5112257957458496, |
| "mean_token_accuracy": 0.8511955315247178, |
| "num_tokens": 86962128.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.513796275202185, |
| "epoch": 6.5081885856079404, |
| "grad_norm": 1.140625, |
| "learning_rate": 5.753990266560105e-06, |
| "loss": 0.5103893280029297, |
| "mean_token_accuracy": 0.8513250714167953, |
| "num_tokens": 87674788.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 0.512695446331054, |
| "epoch": 6.561124896608767, |
| "grad_norm": 1.2109375, |
| "learning_rate": 5.599531525118373e-06, |
| "loss": 0.5133617877960205, |
| "mean_token_accuracy": 0.8500851722434163, |
| "num_tokens": 88377891.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.5171741805970669, |
| "epoch": 6.614061207609595, |
| "grad_norm": 1.15625, |
| "learning_rate": 5.446363971367638e-06, |
| "loss": 0.5159895896911622, |
| "mean_token_accuracy": 0.849970761872828, |
| "num_tokens": 89083739.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.5245028464123607, |
| "epoch": 6.666997518610422, |
| "grad_norm": 1.1640625, |
| "learning_rate": 5.29453254780917e-06, |
| "loss": 0.5201656818389893, |
| "mean_token_accuracy": 0.8486853934824466, |
| "num_tokens": 89799048.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 0.514243483170867, |
| "epoch": 6.719933829611249, |
| "grad_norm": 1.140625, |
| "learning_rate": 5.1440818048962575e-06, |
| "loss": 0.5119954109191894, |
| "mean_token_accuracy": 0.8505460506305098, |
| "num_tokens": 90506283.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 0.5168994429521263, |
| "epoch": 6.772870140612076, |
| "grad_norm": 1.140625, |
| "learning_rate": 4.995055887962167e-06, |
| "loss": 0.514662790298462, |
| "mean_token_accuracy": 0.8501609206199646, |
| "num_tokens": 91217518.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 0.5189123339019716, |
| "epoch": 6.825806451612904, |
| "grad_norm": 1.203125, |
| "learning_rate": 4.8474985242669965e-06, |
| "loss": 0.5177834033966064, |
| "mean_token_accuracy": 0.8493126900866628, |
| "num_tokens": 91921292.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 0.5235454078763724, |
| "epoch": 6.87874276261373, |
| "grad_norm": 1.1875, |
| "learning_rate": 4.701453010167217e-06, |
| "loss": 0.5218941688537597, |
| "mean_token_accuracy": 0.8480734698474407, |
| "num_tokens": 92637986.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.5194550552405417, |
| "epoch": 6.931679073614557, |
| "grad_norm": 1.15625, |
| "learning_rate": 4.556962198411631e-06, |
| "loss": 0.5179105758666992, |
| "mean_token_accuracy": 0.8491211403161287, |
| "num_tokens": 93350635.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 0.5063929574564099, |
| "epoch": 6.984615384615385, |
| "grad_norm": 1.171875, |
| "learning_rate": 4.414068485567473e-06, |
| "loss": 0.5035317897796631, |
| "mean_token_accuracy": 0.8522688586264848, |
| "num_tokens": 94067855.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 0.5093034647804705, |
| "epoch": 7.037055417700579, |
| "grad_norm": 1.1015625, |
| "learning_rate": 4.272813799580417e-06, |
| "loss": 0.5076330184936524, |
| "mean_token_accuracy": 0.8516658600948587, |
| "num_tokens": 94783200.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 0.5048199588432908, |
| "epoch": 7.089991728701406, |
| "grad_norm": 1.15625, |
| "learning_rate": 4.1332395874720474e-06, |
| "loss": 0.5005260467529297, |
| "mean_token_accuracy": 0.85294623170048, |
| "num_tokens": 95493882.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.5057683161459863, |
| "epoch": 7.142928039702233, |
| "grad_norm": 1.1796875, |
| "learning_rate": 3.9953868031784415e-06, |
| "loss": 0.5044948577880859, |
| "mean_token_accuracy": 0.8519550319761038, |
| "num_tokens": 96208176.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.512921197898686, |
| "epoch": 7.195864350703061, |
| "grad_norm": 1.09375, |
| "learning_rate": 3.8592958955334636e-06, |
| "loss": 0.5127240180969238, |
| "mean_token_accuracy": 0.8506218697875738, |
| "num_tokens": 96919756.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.5144638500176371, |
| "epoch": 7.248800661703887, |
| "grad_norm": 1.171875, |
| "learning_rate": 3.725006796400231e-06, |
| "loss": 0.5118832588195801, |
| "mean_token_accuracy": 0.8508855536580086, |
| "num_tokens": 97630930.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.5114422076381743, |
| "epoch": 7.301736972704715, |
| "grad_norm": 1.2109375, |
| "learning_rate": 3.5925589089542954e-06, |
| "loss": 0.5101255893707275, |
| "mean_token_accuracy": 0.8508487151935696, |
| "num_tokens": 98345419.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 0.5078697509132326, |
| "epoch": 7.354673283705542, |
| "grad_norm": 1.1875, |
| "learning_rate": 3.461991096121955e-06, |
| "loss": 0.5053506851196289, |
| "mean_token_accuracy": 0.8518306853249669, |
| "num_tokens": 99059531.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 0.5088249345310032, |
| "epoch": 7.4076095947063685, |
| "grad_norm": 1.1484375, |
| "learning_rate": 3.3333416691770402e-06, |
| "loss": 0.5070612907409668, |
| "mean_token_accuracy": 0.8512004481628537, |
| "num_tokens": 99773516.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.5082653394900263, |
| "epoch": 7.460545905707196, |
| "grad_norm": 1.1875, |
| "learning_rate": 3.206648376499637e-06, |
| "loss": 0.5041531562805176, |
| "mean_token_accuracy": 0.8518870715051889, |
| "num_tokens": 100494087.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.5059876752085983, |
| "epoch": 7.513482216708023, |
| "grad_norm": 1.171875, |
| "learning_rate": 3.0819483924999183e-06, |
| "loss": 0.5048664093017579, |
| "mean_token_accuracy": 0.8521129397675395, |
| "num_tokens": 101214749.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 0.5057537339627742, |
| "epoch": 7.5664185277088505, |
| "grad_norm": 1.140625, |
| "learning_rate": 2.959278306710409e-06, |
| "loss": 0.5029214382171631, |
| "mean_token_accuracy": 0.8521661214530468, |
| "num_tokens": 101926434.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 0.5102150899358093, |
| "epoch": 7.619354838709677, |
| "grad_norm": 1.1328125, |
| "learning_rate": 2.838674113049884e-06, |
| "loss": 0.5092745780944824, |
| "mean_token_accuracy": 0.8507832719013095, |
| "num_tokens": 102638578.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 0.5191028297878801, |
| "epoch": 7.672291149710505, |
| "grad_norm": 1.140625, |
| "learning_rate": 2.7201711992619873e-06, |
| "loss": 0.5185222625732422, |
| "mean_token_accuracy": 0.849537917971611, |
| "num_tokens": 103358219.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.5148415211588144, |
| "epoch": 7.725227460711332, |
| "grad_norm": 1.109375, |
| "learning_rate": 2.6038043365317643e-06, |
| "loss": 0.5099134922027588, |
| "mean_token_accuracy": 0.8513738945126533, |
| "num_tokens": 104064920.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 0.5042823067866266, |
| "epoch": 7.778163771712158, |
| "grad_norm": 1.109375, |
| "learning_rate": 2.489607669283088e-06, |
| "loss": 0.5005523681640625, |
| "mean_token_accuracy": 0.8534011922776699, |
| "num_tokens": 104774666.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.5115070443600416, |
| "epoch": 7.831100082712986, |
| "grad_norm": 1.1484375, |
| "learning_rate": 2.3776147051599518e-06, |
| "loss": 0.5110672950744629, |
| "mean_token_accuracy": 0.8511148374527693, |
| "num_tokens": 105479403.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.5202959575690329, |
| "epoch": 7.884036393713813, |
| "grad_norm": 1.109375, |
| "learning_rate": 2.2678583051946733e-06, |
| "loss": 0.5226257801055908, |
| "mean_token_accuracy": 0.8490624286234378, |
| "num_tokens": 106198229.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.5219967384822667, |
| "epoch": 7.93697270471464, |
| "grad_norm": 1.203125, |
| "learning_rate": 2.1603706741657803e-06, |
| "loss": 0.5207705020904541, |
| "mean_token_accuracy": 0.8481754407286644, |
| "num_tokens": 106903767.0, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.5128119274973869, |
| "epoch": 7.989909015715467, |
| "grad_norm": 1.109375, |
| "learning_rate": 2.055183351148463e-06, |
| "loss": 0.5103421211242676, |
| "mean_token_accuracy": 0.850516995228827, |
| "num_tokens": 107612975.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 0.5038839411284269, |
| "epoch": 8.042349048800661, |
| "grad_norm": 1.125, |
| "learning_rate": 1.9523272002603745e-06, |
| "loss": 0.5004650115966797, |
| "mean_token_accuracy": 0.853232373762582, |
| "num_tokens": 108320998.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 0.5100298066623509, |
| "epoch": 8.095285359801489, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.8518324016054601e-06, |
| "loss": 0.5064547538757325, |
| "mean_token_accuracy": 0.8520022116601467, |
| "num_tokens": 109032408.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.51362031288445, |
| "epoch": 8.148221670802316, |
| "grad_norm": 1.1640625, |
| "learning_rate": 1.7537284424184852e-06, |
| "loss": 0.5118602752685547, |
| "mean_token_accuracy": 0.8505887657403945, |
| "num_tokens": 109748057.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 0.5209537568502128, |
| "epoch": 8.201157981803144, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.658044108412905e-06, |
| "loss": 0.517811393737793, |
| "mean_token_accuracy": 0.8487363727763295, |
| "num_tokens": 110462975.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.5127804886549712, |
| "epoch": 8.25409429280397, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.564807475334509e-06, |
| "loss": 0.5130415439605713, |
| "mean_token_accuracy": 0.8503114173188806, |
| "num_tokens": 111178903.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.5188280244357883, |
| "epoch": 8.307030603804797, |
| "grad_norm": 1.203125, |
| "learning_rate": 1.4740459007234498e-06, |
| "loss": 0.5185968399047851, |
| "mean_token_accuracy": 0.8486667726188898, |
| "num_tokens": 111897536.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 0.5110513370484113, |
| "epoch": 8.359966914805625, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.3857860158869696e-06, |
| "loss": 0.5104970455169677, |
| "mean_token_accuracy": 0.8507039345800876, |
| "num_tokens": 112605102.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 0.515809912700206, |
| "epoch": 8.412903225806451, |
| "grad_norm": 1.125, |
| "learning_rate": 1.3000537180852213e-06, |
| "loss": 0.5132129669189454, |
| "mean_token_accuracy": 0.8504124451428652, |
| "num_tokens": 113325128.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 0.5137974722310901, |
| "epoch": 8.465839536807279, |
| "grad_norm": 1.1171875, |
| "learning_rate": 1.2168741629324944e-06, |
| "loss": 0.5105469226837158, |
| "mean_token_accuracy": 0.8514914209023118, |
| "num_tokens": 114034422.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.49651987878605724, |
| "epoch": 8.518775847808106, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.1362717570160387e-06, |
| "loss": 0.4945973873138428, |
| "mean_token_accuracy": 0.8543671565130353, |
| "num_tokens": 114750388.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 0.5087006632238626, |
| "epoch": 8.571712158808934, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.0582701507346582e-06, |
| "loss": 0.5052749156951905, |
| "mean_token_accuracy": 0.8516966653987765, |
| "num_tokens": 115466046.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 0.5140711238607765, |
| "epoch": 8.62464846980976, |
| "grad_norm": 1.1484375, |
| "learning_rate": 9.828922313592227e-07, |
| "loss": 0.5126253604888916, |
| "mean_token_accuracy": 0.8507738070562482, |
| "num_tokens": 116185312.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.511335510481149, |
| "epoch": 8.677584780810587, |
| "grad_norm": 1.171875, |
| "learning_rate": 9.10160116317047e-07, |
| "loss": 0.5096184253692627, |
| "mean_token_accuracy": 0.8509457372128963, |
| "num_tokens": 116903441.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 0.5210565337911248, |
| "epoch": 8.730521091811415, |
| "grad_norm": 1.15625, |
| "learning_rate": 8.400951467022067e-07, |
| "loss": 0.520634937286377, |
| "mean_token_accuracy": 0.8484291834756732, |
| "num_tokens": 117609009.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.512825894355774, |
| "epoch": 8.78345740281224, |
| "grad_norm": 1.1484375, |
| "learning_rate": 7.727178810136094e-07, |
| "loss": 0.509746503829956, |
| "mean_token_accuracy": 0.8509014787152409, |
| "num_tokens": 118312265.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 0.5040128762833774, |
| "epoch": 8.836393713813068, |
| "grad_norm": 1.109375, |
| "learning_rate": 7.080480891227126e-07, |
| "loss": 0.5000125408172608, |
| "mean_token_accuracy": 0.8538393141701818, |
| "num_tokens": 119025289.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 0.5164961713366211, |
| "epoch": 8.889330024813896, |
| "grad_norm": 1.125, |
| "learning_rate": 6.461047464726433e-07, |
| "loss": 0.5159685134887695, |
| "mean_token_accuracy": 0.8503626996651292, |
| "num_tokens": 119739051.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 0.5125316453166306, |
| "epoch": 8.942266335814724, |
| "grad_norm": 1.1953125, |
| "learning_rate": 5.869060285104089e-07, |
| "loss": 0.512110424041748, |
| "mean_token_accuracy": 0.8507427591830492, |
| "num_tokens": 120447293.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.5029088127426803, |
| "epoch": 8.99520264681555, |
| "grad_norm": 1.0703125, |
| "learning_rate": 5.304693053538434e-07, |
| "loss": 0.5010025978088379, |
| "mean_token_accuracy": 0.8526629090309144, |
| "num_tokens": 121152193.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.5036710680283206, |
| "epoch": 9.047642679900745, |
| "grad_norm": 1.5234375, |
| "learning_rate": 4.768111366948736e-07, |
| "loss": 0.501908826828003, |
| "mean_token_accuracy": 0.8532915790374347, |
| "num_tokens": 121857419.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 0.5134924652986228, |
| "epoch": 9.100578990901571, |
| "grad_norm": 1.25, |
| "learning_rate": 4.2594726694055397e-07, |
| "loss": 0.5120198249816894, |
| "mean_token_accuracy": 0.8511697359383106, |
| "num_tokens": 122583636.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.5101554320193827, |
| "epoch": 9.153515301902399, |
| "grad_norm": 1.140625, |
| "learning_rate": 3.778926205933342e-07, |
| "loss": 0.5077533721923828, |
| "mean_token_accuracy": 0.8517538851127029, |
| "num_tokens": 123297381.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.5169664960354566, |
| "epoch": 9.206451612903226, |
| "grad_norm": 1.1328125, |
| "learning_rate": 3.3266129787191015e-07, |
| "loss": 0.5152172088623047, |
| "mean_token_accuracy": 0.8497943015769124, |
| "num_tokens": 124011276.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.5125159340910613, |
| "epoch": 9.259387923904052, |
| "grad_norm": 1.1640625, |
| "learning_rate": 2.9026657057392715e-07, |
| "loss": 0.5099777221679688, |
| "mean_token_accuracy": 0.851446901448071, |
| "num_tokens": 124728740.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.5071758638136089, |
| "epoch": 9.31232423490488, |
| "grad_norm": 1.1640625, |
| "learning_rate": 2.507208781817638e-07, |
| "loss": 0.5061943531036377, |
| "mean_token_accuracy": 0.8509785430505872, |
| "num_tokens": 125444088.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.5091351515613496, |
| "epoch": 9.365260545905707, |
| "grad_norm": 1.15625, |
| "learning_rate": 2.1403582421253398e-07, |
| "loss": 0.5047785758972168, |
| "mean_token_accuracy": 0.8519781129434705, |
| "num_tokens": 126153025.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.5079080010764301, |
| "epoch": 9.418196856906535, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.8022217281336908e-07, |
| "loss": 0.5062648773193359, |
| "mean_token_accuracy": 0.8518396150320768, |
| "num_tokens": 126874049.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 0.5084890386089682, |
| "epoch": 9.471133167907361, |
| "grad_norm": 1.09375, |
| "learning_rate": 1.4928984560301006e-07, |
| "loss": 0.5026658535003662, |
| "mean_token_accuracy": 0.8528311153873801, |
| "num_tokens": 127587154.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.5195510504767299, |
| "epoch": 9.524069478908189, |
| "grad_norm": 1.1015625, |
| "learning_rate": 1.212479187605897e-07, |
| "loss": 0.5181485652923584, |
| "mean_token_accuracy": 0.8493357080966234, |
| "num_tokens": 128290741.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.5159352781251073, |
| "epoch": 9.577005789909016, |
| "grad_norm": 1.1796875, |
| "learning_rate": 9.610462036249823e-08, |
| "loss": 0.5147523403167724, |
| "mean_token_accuracy": 0.8493468537926674, |
| "num_tokens": 129010702.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.5064713466912508, |
| "epoch": 9.629942100909842, |
| "grad_norm": 1.1328125, |
| "learning_rate": 7.386732796810014e-08, |
| "loss": 0.5044877052307128, |
| "mean_token_accuracy": 0.8523369859904051, |
| "num_tokens": 129724831.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 0.5084437859244645, |
| "epoch": 9.68287841191067, |
| "grad_norm": 1.140625, |
| "learning_rate": 5.454256645500477e-08, |
| "loss": 0.5059144973754883, |
| "mean_token_accuracy": 0.8517411896958947, |
| "num_tokens": 130432884.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.518665280751884, |
| "epoch": 9.735814722911497, |
| "grad_norm": 1.1484375, |
| "learning_rate": 3.8136006104533276e-08, |
| "loss": 0.5166204452514649, |
| "mean_token_accuracy": 0.8493583101779223, |
| "num_tokens": 131154186.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 0.5102572640404105, |
| "epoch": 9.788751033912325, |
| "grad_norm": 1.1796875, |
| "learning_rate": 2.4652460937941714e-08, |
| "loss": 0.5090443134307862, |
| "mean_token_accuracy": 0.8509866768494249, |
| "num_tokens": 131867648.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.5133054099045694, |
| "epoch": 9.84168734491315, |
| "grad_norm": 1.125, |
| "learning_rate": 1.4095887303888778e-08, |
| "loss": 0.5111370086669922, |
| "mean_token_accuracy": 0.8502802401781082, |
| "num_tokens": 132562489.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 0.5081644008867443, |
| "epoch": 9.894623655913978, |
| "grad_norm": 1.1796875, |
| "learning_rate": 6.469382717563255e-09, |
| "loss": 0.5059957027435302, |
| "mean_token_accuracy": 0.8521056715399027, |
| "num_tokens": 133272125.0, |
| "step": 1870 |
| }, |
| { |
| "entropy": 0.5070097131654621, |
| "epoch": 9.947559966914806, |
| "grad_norm": 1.15625, |
| "learning_rate": 1.775184951806619e-09, |
| "loss": 0.505557632446289, |
| "mean_token_accuracy": 0.8517521150410176, |
| "num_tokens": 133981204.0, |
| "step": 1880 |
| }, |
| { |
| "entropy": 0.5107753489295767, |
| "epoch": 10.0, |
| "grad_norm": 1.234375, |
| "learning_rate": 1.4671380509367893e-11, |
| "loss": 0.509831428527832, |
| "mean_token_accuracy": 0.8506396452335153, |
| "num_tokens": 134689470.0, |
| "step": 1890 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1890, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.6360874475383685e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
