Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN,
"... is not valid JSON
| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.20743525751142514, | |
| "eval_steps": 500, | |
| "global_step": 400, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.005185881437785629, | |
| "grad_norm": 247.97032165527344, | |
| "learning_rate": 2.5499999999999996e-05, | |
| "loss": 145.23812255859374, | |
| "original_ce_loss": 9.0141, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.010371762875571258, | |
| "grad_norm": 248.38844299316406, | |
| "learning_rate": 6.8e-05, | |
| "loss": 144.60938720703126, | |
| "original_ce_loss": 8.9749, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.015557644313356886, | |
| "grad_norm": 385.3697204589844, | |
| "learning_rate": 0.00010625, | |
| "loss": 141.92376708984375, | |
| "original_ce_loss": 8.8072, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.020743525751142515, | |
| "grad_norm": 250.706787109375, | |
| "learning_rate": 0.00014450000000000002, | |
| "loss": 129.28629150390626, | |
| "original_ce_loss": 8.0175, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.025929407188928142, | |
| "grad_norm": 37.78219985961914, | |
| "learning_rate": 0.00018275, | |
| "loss": 118.50413818359375, | |
| "original_ce_loss": 7.3439, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.031115288626713773, | |
| "grad_norm": 35.401676177978516, | |
| "learning_rate": 0.00022525, | |
| "loss": 112.6171630859375, | |
| "original_ce_loss": 6.9763, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.0363011700644994, | |
| "grad_norm": 155.93934631347656, | |
| "learning_rate": 0.00026775, | |
| "loss": 105.485986328125, | |
| "original_ce_loss": 6.5312, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.04148705150228503, | |
| "grad_norm": 17.077312469482422, | |
| "learning_rate": 0.00031025, | |
| "loss": 102.00914916992187, | |
| "original_ce_loss": 6.3146, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.04667293294007066, | |
| "grad_norm": 12.143115997314453, | |
| "learning_rate": 0.00035275, | |
| "loss": 101.21661376953125, | |
| "original_ce_loss": 6.2657, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.051858814377856284, | |
| "grad_norm": 212.8907928466797, | |
| "learning_rate": 0.00039525, | |
| "loss": 101.29624633789062, | |
| "original_ce_loss": 6.2715, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.057044695815641915, | |
| "grad_norm": 379.0343933105469, | |
| "learning_rate": 0.00042499874610596206, | |
| "loss": 100.54356689453125, | |
| "original_ce_loss": 6.225, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.062230577253427545, | |
| "grad_norm": 119.57896423339844, | |
| "learning_rate": 0.000424962070796437, | |
| "loss": 99.99188232421875, | |
| "original_ce_loss": 6.1911, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.06741645869121317, | |
| "grad_norm": 176.70579528808594, | |
| "learning_rate": 0.0004248746228038075, | |
| "loss": 99.34671630859376, | |
| "original_ce_loss": 6.1514, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.0726023401289988, | |
| "grad_norm": 59.76472473144531, | |
| "learning_rate": 0.0004247179359861924, | |
| "loss": 99.34248046875, | |
| "original_ce_loss": 6.1519, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.07778822156678443, | |
| "grad_norm": 293.2443542480469, | |
| "learning_rate": 0.0004245475044883495, | |
| "loss": 99.488525390625, | |
| "original_ce_loss": 6.1615, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.08297410300457006, | |
| "grad_norm": 714.7462158203125, | |
| "learning_rate": 0.0004243079123379424, | |
| "loss": 99.61160278320312, | |
| "original_ce_loss": 6.1695, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.08815998444235569, | |
| "grad_norm": 886.433837890625, | |
| "learning_rate": 0.0004239823368702875, | |
| "loss": 99.01062622070313, | |
| "original_ce_loss": 6.1322, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.09334586588014132, | |
| "grad_norm": 200.77587890625, | |
| "learning_rate": 0.0004235943684185158, | |
| "loss": 98.5961181640625, | |
| "original_ce_loss": 6.1064, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.09853174731792694, | |
| "grad_norm": 48.907493591308594, | |
| "learning_rate": 0.0004231441214437677, | |
| "loss": 98.40054931640626, | |
| "original_ce_loss": 6.0944, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.10371762875571257, | |
| "grad_norm": 78.08061981201172, | |
| "learning_rate": 0.00042263172878102487, | |
| "loss": 98.215380859375, | |
| "original_ce_loss": 6.0832, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.1089035101934982, | |
| "grad_norm": 123.57373046875, | |
| "learning_rate": 0.00042205734159992043, | |
| "loss": 98.25463256835937, | |
| "original_ce_loss": 6.0864, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.11408939163128383, | |
| "grad_norm": 394.4930419921875, | |
| "learning_rate": 0.0004214875275050289, | |
| "loss": 98.41907348632813, | |
| "original_ce_loss": 6.0973, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.11927527306906946, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00042086777104881595, | |
| "loss": 98.69362182617188, | |
| "original_ce_loss": 6.115, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.12446115450685509, | |
| "grad_norm": 35506.96484375, | |
| "learning_rate": 0.0004202750689631276, | |
| "loss": 99.12110595703125, | |
| "original_ce_loss": 6.1421, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.12964703594464072, | |
| "grad_norm": 11162.1083984375, | |
| "learning_rate": 0.00041964313501643334, | |
| "loss": 99.37658081054687, | |
| "original_ce_loss": 6.1583, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.13483291738242634, | |
| "grad_norm": 1199.7325439453125, | |
| "learning_rate": 0.00041879823026024296, | |
| "loss": 99.02756958007812, | |
| "original_ce_loss": 6.1366, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.14001879882021198, | |
| "grad_norm": 33510.7578125, | |
| "learning_rate": 0.0004178924619709821, | |
| "loss": 98.757275390625, | |
| "original_ce_loss": 6.1199, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.1452046802579976, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.0004170254526824216, | |
| "loss": 98.8090087890625, | |
| "original_ce_loss": 6.1234, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.15039056169578324, | |
| "grad_norm": 7444.04150390625, | |
| "learning_rate": 0.0004161095673476243, | |
| "loss": 100.40501098632812, | |
| "original_ce_loss": 6.2235, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.15557644313356886, | |
| "grad_norm": 5086.20166015625, | |
| "learning_rate": 0.0004151450248383901, | |
| "loss": 102.10885009765624, | |
| "original_ce_loss": 6.3303, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.16076232457135448, | |
| "grad_norm": 2572.272216796875, | |
| "learning_rate": 0.00041401652462566313, | |
| "loss": 101.41973876953125, | |
| "original_ce_loss": 6.2876, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.16594820600914012, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00041282857161178824, | |
| "loss": 101.670556640625, | |
| "original_ce_loss": 6.3039, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.17113408744692574, | |
| "grad_norm": 25727.53515625, | |
| "learning_rate": 0.0004118356381184281, | |
| "loss": 103.318212890625, | |
| "original_ce_loss": 6.4075, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.17631996888471138, | |
| "grad_norm": 35157.0859375, | |
| "learning_rate": 0.00041054156498848937, | |
| "loss": 110.7964599609375, | |
| "original_ce_loss": 6.8758, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.181505850322497, | |
| "grad_norm": 112891.84375, | |
| "learning_rate": 0.0004091890642640685, | |
| "loss": 126.60723876953125, | |
| "original_ce_loss": 7.8646, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.18669173176028264, | |
| "grad_norm": 3006.623779296875, | |
| "learning_rate": 0.000407778534969299, | |
| "loss": 119.78572998046874, | |
| "original_ce_loss": 7.4386, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.19187761319806826, | |
| "grad_norm": 2020.123779296875, | |
| "learning_rate": 0.0004063103932483052, | |
| "loss": 114.20302734375, | |
| "original_ce_loss": 7.0899, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.19706349463585388, | |
| "grad_norm": 9304.6865234375, | |
| "learning_rate": 0.0004047850722424284, | |
| "loss": 108.25167236328124, | |
| "original_ce_loss": 6.7183, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.20224937607363952, | |
| "grad_norm": 6329.39794921875, | |
| "learning_rate": 0.0004032030219624389, | |
| "loss": 105.06466064453124, | |
| "original_ce_loss": 6.5199, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.20743525751142514, | |
| "grad_norm": 852.490478515625, | |
| "learning_rate": 0.0004015647091557703, | |
| "loss": 105.2643798828125, | |
| "original_ce_loss": 6.5331, | |
| "step": 400 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1929, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 200, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 4249361190838272.0, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |