aixk's picture
Auto-upload checkpoint step 800
36bea6b verified
Raw History Blame Contribute Delete
15.5 kB
Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN, "... is not valid JSON
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.00467445352716734,
"eval_steps": 500,
"global_step": 800,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 5.8430669089591743e-05,
"grad_norm": 10.995234489440918,
"learning_rate": 6e-06,
"loss": 36.76142578125,
"step": 10
},
{
"epoch": 0.00011686133817918349,
"grad_norm": 10.91879653930664,
"learning_rate": 1.2666666666666668e-05,
"loss": 36.63887023925781,
"step": 20
},
{
"epoch": 0.00017529200726877522,
"grad_norm": 12.004676818847656,
"learning_rate": 1.9333333333333333e-05,
"loss": 36.38578491210937,
"step": 30
},
{
"epoch": 0.00023372267635836697,
"grad_norm": 11.3691987991333,
"learning_rate": 2.6000000000000002e-05,
"loss": 36.06534118652344,
"step": 40
},
{
"epoch": 0.0002921533454479587,
"grad_norm": 13.08530044555664,
"learning_rate": 3.266666666666667e-05,
"loss": 35.64937744140625,
"step": 50
},
{
"epoch": 0.00035058401453755045,
"grad_norm": 13.18602180480957,
"learning_rate": 3.933333333333333e-05,
"loss": 35.051446533203126,
"step": 60
},
{
"epoch": 0.0004090146836271422,
"grad_norm": 13.574982643127441,
"learning_rate": 4.600000000000001e-05,
"loss": 34.180169677734376,
"step": 70
},
{
"epoch": 0.00046744535271673395,
"grad_norm": 14.570645332336426,
"learning_rate": 5.266666666666666e-05,
"loss": 33.2185546875,
"step": 80
},
{
"epoch": 0.0005258760218063257,
"grad_norm": 16.55890464782715,
"learning_rate": 5.9333333333333343e-05,
"loss": 32.12116394042969,
"step": 90
},
{
"epoch": 0.0005843066908959174,
"grad_norm": 17.331336975097656,
"learning_rate": 6.6e-05,
"loss": 30.47571716308594,
"step": 100
},
{
"epoch": 0.0006427373599855092,
"grad_norm": 14.68326187133789,
"learning_rate": 7.266666666666667e-05,
"loss": 28.728692626953126,
"step": 110
},
{
"epoch": 0.0007011680290751009,
"grad_norm": 18.396814346313477,
"learning_rate": 7.933333333333334e-05,
"loss": 26.857958984375,
"step": 120
},
{
"epoch": 0.0007595986981646927,
"grad_norm": 16.36812973022461,
"learning_rate": 8.6e-05,
"loss": 25.132843017578125,
"step": 130
},
{
"epoch": 0.0008180293672542844,
"grad_norm": 21.44991111755371,
"learning_rate": 9.266666666666666e-05,
"loss": 23.530233764648436,
"step": 140
},
{
"epoch": 0.0008764600363438762,
"grad_norm": 22.47516632080078,
"learning_rate": 9.933333333333334e-05,
"loss": 22.332640075683592,
"step": 150
},
{
"epoch": 0.0009348907054334679,
"grad_norm": 22.053213119506836,
"learning_rate": 0.00010600000000000002,
"loss": 21.743748474121094,
"step": 160
},
{
"epoch": 0.0009933213745230596,
"grad_norm": 26.53052520751953,
"learning_rate": 0.00011266666666666668,
"loss": 20.7405029296875,
"step": 170
},
{
"epoch": 0.0010517520436126514,
"grad_norm": 37.71436309814453,
"learning_rate": 0.00011933333333333334,
"loss": 19.994790649414064,
"step": 180
},
{
"epoch": 0.001110182712702243,
"grad_norm": 33.68248748779297,
"learning_rate": 0.000126,
"loss": 19.59712371826172,
"step": 190
},
{
"epoch": 0.001168613381791835,
"grad_norm": 29.311405181884766,
"learning_rate": 0.00013266666666666667,
"loss": 19.54248809814453,
"step": 200
},
{
"epoch": 0.0012270440508814267,
"grad_norm": 27.571697235107422,
"learning_rate": 0.00013933333333333335,
"loss": 18.915852355957032,
"step": 210
},
{
"epoch": 0.0012854747199710183,
"grad_norm": 50.6077880859375,
"learning_rate": 0.000146,
"loss": 18.705543518066406,
"step": 220
},
{
"epoch": 0.0013439053890606102,
"grad_norm": 24.939199447631836,
"learning_rate": 0.00015266666666666667,
"loss": 19.156636047363282,
"step": 230
},
{
"epoch": 0.0014023360581502018,
"grad_norm": 28.88422393798828,
"learning_rate": 0.00015933333333333332,
"loss": 17.941325378417968,
"step": 240
},
{
"epoch": 0.0014607667272397936,
"grad_norm": 32.721168518066406,
"learning_rate": 0.000166,
"loss": 17.255938720703124,
"step": 250
},
{
"epoch": 0.0015191973963293855,
"grad_norm": 44.036075592041016,
"learning_rate": 0.00017266666666666667,
"loss": 17.748182678222655,
"step": 260
},
{
"epoch": 0.001577628065418977,
"grad_norm": 30.866708755493164,
"learning_rate": 0.00017933333333333332,
"loss": 17.471981811523438,
"step": 270
},
{
"epoch": 0.001636058734508569,
"grad_norm": 58.11023712158203,
"learning_rate": 0.00018600000000000002,
"loss": 17.05304870605469,
"step": 280
},
{
"epoch": 0.0016944894035981605,
"grad_norm": 94.0334243774414,
"learning_rate": 0.0001926666666666667,
"loss": 16.84721221923828,
"step": 290
},
{
"epoch": 0.0017529200726877523,
"grad_norm": 75.26679992675781,
"learning_rate": 0.00019933333333333334,
"loss": 16.655764770507812,
"step": 300
},
{
"epoch": 0.0018113507417773442,
"grad_norm": NaN,
"learning_rate": 0.00019999999996000412,
"loss": 17.352436828613282,
"step": 310
},
{
"epoch": 0.0018697814108669358,
"grad_norm": NaN,
"learning_rate": 0.00019999999982174674,
"loss": 17.325209045410155,
"step": 320
},
{
"epoch": 0.0019282120799565276,
"grad_norm": 421.18212890625,
"learning_rate": 0.00019999999958473402,
"loss": 17.76262512207031,
"step": 330
},
{
"epoch": 0.0019866427490461192,
"grad_norm": Infinity,
"learning_rate": 0.00019999999924896604,
"loss": 17.62756805419922,
"step": 340
},
{
"epoch": 0.002045073418135711,
"grad_norm": 128304.0234375,
"learning_rate": 0.00019999999881444277,
"loss": 18.896243286132812,
"step": 350
},
{
"epoch": 0.002103504087225303,
"grad_norm": 1544019.375,
"learning_rate": 0.00019999999828116422,
"loss": 19.169711303710937,
"step": 360
},
{
"epoch": 0.0021619347563148947,
"grad_norm": 2917.049072265625,
"learning_rate": 0.00019999999764913037,
"loss": 19.900315856933595,
"step": 370
},
{
"epoch": 0.002220365425404486,
"grad_norm": 507.0084533691406,
"learning_rate": 0.00019999999691834125,
"loss": 20.770042419433594,
"step": 380
},
{
"epoch": 0.002278796094494078,
"grad_norm": 6932.96435546875,
"learning_rate": 0.00019999999608879683,
"loss": 21.493869018554687,
"step": 390
},
{
"epoch": 0.00233722676358367,
"grad_norm": 711.9380493164062,
"learning_rate": 0.00019999999516049713,
"loss": 21.287228393554688,
"step": 400
},
{
"epoch": 0.0023956574326732616,
"grad_norm": 2512.957275390625,
"learning_rate": 0.00019999999413344217,
"loss": 20.687088012695312,
"step": 410
},
{
"epoch": 0.0024540881017628535,
"grad_norm": 245.29962158203125,
"learning_rate": 0.0001999999930076319,
"loss": 19.545924377441406,
"step": 420
},
{
"epoch": 0.002512518770852445,
"grad_norm": 873.497314453125,
"learning_rate": 0.00019999999178306637,
"loss": 19.846446228027343,
"step": 430
},
{
"epoch": 0.0025709494399420367,
"grad_norm": 273.556884765625,
"learning_rate": 0.00019999999045974555,
"loss": 19.4821044921875,
"step": 440
},
{
"epoch": 0.0026293801090316285,
"grad_norm": 105.1680679321289,
"learning_rate": 0.00019999998903766944,
"loss": 18.83043212890625,
"step": 450
},
{
"epoch": 0.0026878107781212203,
"grad_norm": 19779.220703125,
"learning_rate": 0.00019999998751683804,
"loss": 18.68498992919922,
"step": 460
},
{
"epoch": 0.002746241447210812,
"grad_norm": 379.18695068359375,
"learning_rate": 0.0001999999858972514,
"loss": 17.952909851074217,
"step": 470
},
{
"epoch": 0.0028046721163004036,
"grad_norm": 982.7449340820312,
"learning_rate": 0.00019999998417890945,
"loss": 18.662619018554686,
"step": 480
},
{
"epoch": 0.0028631027853899954,
"grad_norm": 264.763671875,
"learning_rate": 0.00019999998236181225,
"loss": 18.683988952636717,
"step": 490
},
{
"epoch": 0.0029215334544795872,
"grad_norm": 1355.505126953125,
"learning_rate": 0.0001999999804459598,
"loss": 18.34948272705078,
"step": 500
},
{
"epoch": 0.002979964123569179,
"grad_norm": 106.23206329345703,
"learning_rate": 0.00019999997843135202,
"loss": 18.03973388671875,
"step": 510
},
{
"epoch": 0.003038394792658771,
"grad_norm": 135.76995849609375,
"learning_rate": 0.00019999997631798904,
"loss": 18.432786560058595,
"step": 520
},
{
"epoch": 0.0030968254617483623,
"grad_norm": 683.4022216796875,
"learning_rate": 0.00019999997410587078,
"loss": 19.00628662109375,
"step": 530
},
{
"epoch": 0.003155256130837954,
"grad_norm": 1766.309814453125,
"learning_rate": 0.00019999997179499722,
"loss": 18.93055877685547,
"step": 540
},
{
"epoch": 0.003213686799927546,
"grad_norm": 12474.607421875,
"learning_rate": 0.0001999999693853684,
"loss": 19.35320129394531,
"step": 550
},
{
"epoch": 0.003272117469017138,
"grad_norm": 14713.984375,
"learning_rate": 0.00019999996687698437,
"loss": 19.091952514648437,
"step": 560
},
{
"epoch": 0.0033305481381067296,
"grad_norm": 37941.45703125,
"learning_rate": 0.00019999996426984504,
"loss": 18.91400604248047,
"step": 570
},
{
"epoch": 0.003388978807196321,
"grad_norm": 40131.44140625,
"learning_rate": 0.0001999999615639505,
"loss": 19.18461456298828,
"step": 580
},
{
"epoch": 0.003447409476285913,
"grad_norm": 4507.1064453125,
"learning_rate": 0.00019999995875930068,
"loss": 19.337628173828126,
"step": 590
},
{
"epoch": 0.0035058401453755047,
"grad_norm": 69082.234375,
"learning_rate": 0.00019999995585589562,
"loss": 19.456033325195314,
"step": 600
},
{
"epoch": 0.0035642708144650965,
"grad_norm": 3494.4580078125,
"learning_rate": 0.0001999999528537353,
"loss": 19.609593200683594,
"step": 610
},
{
"epoch": 0.0036227014835546884,
"grad_norm": 66.80265808105469,
"learning_rate": 0.00019999994975281976,
"loss": 19.908061218261718,
"step": 620
},
{
"epoch": 0.0036811321526442797,
"grad_norm": 292724.375,
"learning_rate": 0.00019999994655314898,
"loss": 19.095835876464843,
"step": 630
},
{
"epoch": 0.0037395628217338716,
"grad_norm": 27531.40625,
"learning_rate": 0.00019999994325472297,
"loss": 18.935963439941407,
"step": 640
},
{
"epoch": 0.0037979934908234634,
"grad_norm": 82.52088165283203,
"learning_rate": 0.0001999999398575417,
"loss": 18.101783752441406,
"step": 650
},
{
"epoch": 0.0038564241599130552,
"grad_norm": 654071.125,
"learning_rate": 0.0001999999363616052,
"loss": 19.08836212158203,
"step": 660
},
{
"epoch": 0.003914854829002647,
"grad_norm": 2386539.75,
"learning_rate": 0.00019999993276691348,
"loss": 18.599899291992188,
"step": 670
},
{
"epoch": 0.0039732854980922385,
"grad_norm": 54.48868179321289,
"learning_rate": 0.00019999992907346656,
"loss": 18.787478637695312,
"step": 680
},
{
"epoch": 0.00403171616718183,
"grad_norm": 465.9234924316406,
"learning_rate": 0.00019999992528126442,
"loss": 18.208659362792968,
"step": 690
},
{
"epoch": 0.004090146836271422,
"grad_norm": 66.7064437866211,
"learning_rate": 0.00019999992139030704,
"loss": 18.39374542236328,
"step": 700
},
{
"epoch": 0.004148577505361014,
"grad_norm": 86624.9921875,
"learning_rate": 0.00019999991740059448,
"loss": 18.63762969970703,
"step": 710
},
{
"epoch": 0.004207008174450606,
"grad_norm": 79.30855560302734,
"learning_rate": 0.00019999991331212669,
"loss": 19.043922424316406,
"step": 720
},
{
"epoch": 0.004265438843540198,
"grad_norm": 275867.625,
"learning_rate": 0.00019999990912490372,
"loss": 18.727554321289062,
"step": 730
},
{
"epoch": 0.0043238695126297895,
"grad_norm": 2455.1982421875,
"learning_rate": 0.00019999990483892551,
"loss": 18.933006286621094,
"step": 740
},
{
"epoch": 0.004382300181719381,
"grad_norm": 89.4070816040039,
"learning_rate": 0.00019999990045419213,
"loss": 18.374476623535156,
"step": 750
},
{
"epoch": 0.004440730850808972,
"grad_norm": 82.21696472167969,
"learning_rate": 0.00019999989597070357,
"loss": 18.830426025390626,
"step": 760
},
{
"epoch": 0.004499161519898564,
"grad_norm": 92.64002990722656,
"learning_rate": 0.00019999989138845984,
"loss": 19.133992004394532,
"step": 770
},
{
"epoch": 0.004557592188988156,
"grad_norm": 71.72074127197266,
"learning_rate": 0.00019999988670746087,
"loss": 18.759793090820313,
"step": 780
},
{
"epoch": 0.004616022858077748,
"grad_norm": 69.16524505615234,
"learning_rate": 0.00019999988192770677,
"loss": 18.654483032226562,
"step": 790
},
{
"epoch": 0.00467445352716734,
"grad_norm": NaN,
"learning_rate": 0.0001999998770491975,
"loss": 18.783712768554686,
"step": 800
}
],
"logging_steps": 10,
"max_steps": 1000000,
"num_input_tokens_seen": 0,
"num_train_epochs": 6,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 6418641416217600.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}