aixk's picture
Auto-upload checkpoint step 400
d9ca236 verified
Raw History Blame Contribute Delete
8.13 kB
Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN, "... is not valid JSON
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.00233722676358367,
"eval_steps": 500,
"global_step": 400,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 5.8430669089591743e-05,
"grad_norm": 10.995234489440918,
"learning_rate": 6e-06,
"loss": 36.76142578125,
"step": 10
},
{
"epoch": 0.00011686133817918349,
"grad_norm": 10.91879653930664,
"learning_rate": 1.2666666666666668e-05,
"loss": 36.63887023925781,
"step": 20
},
{
"epoch": 0.00017529200726877522,
"grad_norm": 12.004676818847656,
"learning_rate": 1.9333333333333333e-05,
"loss": 36.38578491210937,
"step": 30
},
{
"epoch": 0.00023372267635836697,
"grad_norm": 11.3691987991333,
"learning_rate": 2.6000000000000002e-05,
"loss": 36.06534118652344,
"step": 40
},
{
"epoch": 0.0002921533454479587,
"grad_norm": 13.08530044555664,
"learning_rate": 3.266666666666667e-05,
"loss": 35.64937744140625,
"step": 50
},
{
"epoch": 0.00035058401453755045,
"grad_norm": 13.18602180480957,
"learning_rate": 3.933333333333333e-05,
"loss": 35.051446533203126,
"step": 60
},
{
"epoch": 0.0004090146836271422,
"grad_norm": 13.574982643127441,
"learning_rate": 4.600000000000001e-05,
"loss": 34.180169677734376,
"step": 70
},
{
"epoch": 0.00046744535271673395,
"grad_norm": 14.570645332336426,
"learning_rate": 5.266666666666666e-05,
"loss": 33.2185546875,
"step": 80
},
{
"epoch": 0.0005258760218063257,
"grad_norm": 16.55890464782715,
"learning_rate": 5.9333333333333343e-05,
"loss": 32.12116394042969,
"step": 90
},
{
"epoch": 0.0005843066908959174,
"grad_norm": 17.331336975097656,
"learning_rate": 6.6e-05,
"loss": 30.47571716308594,
"step": 100
},
{
"epoch": 0.0006427373599855092,
"grad_norm": 14.68326187133789,
"learning_rate": 7.266666666666667e-05,
"loss": 28.728692626953126,
"step": 110
},
{
"epoch": 0.0007011680290751009,
"grad_norm": 18.396814346313477,
"learning_rate": 7.933333333333334e-05,
"loss": 26.857958984375,
"step": 120
},
{
"epoch": 0.0007595986981646927,
"grad_norm": 16.36812973022461,
"learning_rate": 8.6e-05,
"loss": 25.132843017578125,
"step": 130
},
{
"epoch": 0.0008180293672542844,
"grad_norm": 21.44991111755371,
"learning_rate": 9.266666666666666e-05,
"loss": 23.530233764648436,
"step": 140
},
{
"epoch": 0.0008764600363438762,
"grad_norm": 22.47516632080078,
"learning_rate": 9.933333333333334e-05,
"loss": 22.332640075683592,
"step": 150
},
{
"epoch": 0.0009348907054334679,
"grad_norm": 22.053213119506836,
"learning_rate": 0.00010600000000000002,
"loss": 21.743748474121094,
"step": 160
},
{
"epoch": 0.0009933213745230596,
"grad_norm": 26.53052520751953,
"learning_rate": 0.00011266666666666668,
"loss": 20.7405029296875,
"step": 170
},
{
"epoch": 0.0010517520436126514,
"grad_norm": 37.71436309814453,
"learning_rate": 0.00011933333333333334,
"loss": 19.994790649414064,
"step": 180
},
{
"epoch": 0.001110182712702243,
"grad_norm": 33.68248748779297,
"learning_rate": 0.000126,
"loss": 19.59712371826172,
"step": 190
},
{
"epoch": 0.001168613381791835,
"grad_norm": 29.311405181884766,
"learning_rate": 0.00013266666666666667,
"loss": 19.54248809814453,
"step": 200
},
{
"epoch": 0.0012270440508814267,
"grad_norm": 27.571697235107422,
"learning_rate": 0.00013933333333333335,
"loss": 18.915852355957032,
"step": 210
},
{
"epoch": 0.0012854747199710183,
"grad_norm": 50.6077880859375,
"learning_rate": 0.000146,
"loss": 18.705543518066406,
"step": 220
},
{
"epoch": 0.0013439053890606102,
"grad_norm": 24.939199447631836,
"learning_rate": 0.00015266666666666667,
"loss": 19.156636047363282,
"step": 230
},
{
"epoch": 0.0014023360581502018,
"grad_norm": 28.88422393798828,
"learning_rate": 0.00015933333333333332,
"loss": 17.941325378417968,
"step": 240
},
{
"epoch": 0.0014607667272397936,
"grad_norm": 32.721168518066406,
"learning_rate": 0.000166,
"loss": 17.255938720703124,
"step": 250
},
{
"epoch": 0.0015191973963293855,
"grad_norm": 44.036075592041016,
"learning_rate": 0.00017266666666666667,
"loss": 17.748182678222655,
"step": 260
},
{
"epoch": 0.001577628065418977,
"grad_norm": 30.866708755493164,
"learning_rate": 0.00017933333333333332,
"loss": 17.471981811523438,
"step": 270
},
{
"epoch": 0.001636058734508569,
"grad_norm": 58.11023712158203,
"learning_rate": 0.00018600000000000002,
"loss": 17.05304870605469,
"step": 280
},
{
"epoch": 0.0016944894035981605,
"grad_norm": 94.0334243774414,
"learning_rate": 0.0001926666666666667,
"loss": 16.84721221923828,
"step": 290
},
{
"epoch": 0.0017529200726877523,
"grad_norm": 75.26679992675781,
"learning_rate": 0.00019933333333333334,
"loss": 16.655764770507812,
"step": 300
},
{
"epoch": 0.0018113507417773442,
"grad_norm": NaN,
"learning_rate": 0.00019999999996000412,
"loss": 17.352436828613282,
"step": 310
},
{
"epoch": 0.0018697814108669358,
"grad_norm": NaN,
"learning_rate": 0.00019999999982174674,
"loss": 17.325209045410155,
"step": 320
},
{
"epoch": 0.0019282120799565276,
"grad_norm": 421.18212890625,
"learning_rate": 0.00019999999958473402,
"loss": 17.76262512207031,
"step": 330
},
{
"epoch": 0.0019866427490461192,
"grad_norm": Infinity,
"learning_rate": 0.00019999999924896604,
"loss": 17.62756805419922,
"step": 340
},
{
"epoch": 0.002045073418135711,
"grad_norm": 128304.0234375,
"learning_rate": 0.00019999999881444277,
"loss": 18.896243286132812,
"step": 350
},
{
"epoch": 0.002103504087225303,
"grad_norm": 1544019.375,
"learning_rate": 0.00019999999828116422,
"loss": 19.169711303710937,
"step": 360
},
{
"epoch": 0.0021619347563148947,
"grad_norm": 2917.049072265625,
"learning_rate": 0.00019999999764913037,
"loss": 19.900315856933595,
"step": 370
},
{
"epoch": 0.002220365425404486,
"grad_norm": 507.0084533691406,
"learning_rate": 0.00019999999691834125,
"loss": 20.770042419433594,
"step": 380
},
{
"epoch": 0.002278796094494078,
"grad_norm": 6932.96435546875,
"learning_rate": 0.00019999999608879683,
"loss": 21.493869018554687,
"step": 390
},
{
"epoch": 0.00233722676358367,
"grad_norm": 711.9380493164062,
"learning_rate": 0.00019999999516049713,
"loss": 21.287228393554688,
"step": 400
}
],
"logging_steps": 10,
"max_steps": 1000000,
"num_input_tokens_seen": 0,
"num_train_epochs": 6,
"save_steps": 200,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 3201253176450048.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}