Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN,
"... is not valid JSON
| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.02, | |
| "eval_steps": 500, | |
| "global_step": 600, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0003333333333333333, | |
| "grad_norm": 243.78553771972656, | |
| "learning_rate": 5.95e-06, | |
| "loss": 166.0732177734375, | |
| "original_ce_loss": 10.3789, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.0006666666666666666, | |
| "grad_norm": 239.2476348876953, | |
| "learning_rate": 1.445e-05, | |
| "loss": 165.82977294921875, | |
| "original_ce_loss": 10.3637, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.001, | |
| "grad_norm": 244.72525024414062, | |
| "learning_rate": 2.295e-05, | |
| "loss": 165.18355712890624, | |
| "original_ce_loss": 10.3233, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.0013333333333333333, | |
| "grad_norm": 270.7417297363281, | |
| "learning_rate": 3.145e-05, | |
| "loss": 163.79927978515624, | |
| "original_ce_loss": 10.2368, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0016666666666666668, | |
| "grad_norm": 326.618408203125, | |
| "learning_rate": 3.9949999999999995e-05, | |
| "loss": 160.3599365234375, | |
| "original_ce_loss": 10.0218, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.002, | |
| "grad_norm": 166.10595703125, | |
| "learning_rate": 4.845e-05, | |
| "loss": 151.0793212890625, | |
| "original_ce_loss": 9.4418, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.0023333333333333335, | |
| "grad_norm": 54.51128387451172, | |
| "learning_rate": 5.695e-05, | |
| "loss": 142.18406982421874, | |
| "original_ce_loss": 8.8858, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.0026666666666666666, | |
| "grad_norm": 45.69681930541992, | |
| "learning_rate": 6.544999999999999e-05, | |
| "loss": 135.513818359375, | |
| "original_ce_loss": 8.4689, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.003, | |
| "grad_norm": 44.38578414916992, | |
| "learning_rate": 7.395e-05, | |
| "loss": 129.2341796875, | |
| "original_ce_loss": 8.0765, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.0033333333333333335, | |
| "grad_norm": 36.00300979614258, | |
| "learning_rate": 8.245e-05, | |
| "loss": 123.0133544921875, | |
| "original_ce_loss": 7.6877, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.0036666666666666666, | |
| "grad_norm": 32.789207458496094, | |
| "learning_rate": 9.094999999999999e-05, | |
| "loss": 117.155908203125, | |
| "original_ce_loss": 7.3216, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.004, | |
| "grad_norm": 26.692514419555664, | |
| "learning_rate": 9.86e-05, | |
| "loss": 112.15670166015624, | |
| "original_ce_loss": 7.0091, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.004333333333333333, | |
| "grad_norm": 20.985740661621094, | |
| "learning_rate": 0.0001071, | |
| "loss": 108.44638671875, | |
| "original_ce_loss": 6.7772, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.004666666666666667, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00011475, | |
| "loss": 105.64476318359375, | |
| "original_ce_loss": 6.6021, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.005, | |
| "grad_norm": 47.759765625, | |
| "learning_rate": 0.000119, | |
| "loss": 104.857763671875, | |
| "original_ce_loss": 6.5529, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.005333333333333333, | |
| "grad_norm": 164.30889892578125, | |
| "learning_rate": 0.00012749999999999998, | |
| "loss": 103.3579833984375, | |
| "original_ce_loss": 6.4592, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.005666666666666667, | |
| "grad_norm": 11.574602127075195, | |
| "learning_rate": 0.000136, | |
| "loss": 103.28526611328125, | |
| "original_ce_loss": 6.4547, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.006, | |
| "grad_norm": 13.660222053527832, | |
| "learning_rate": 0.00014450000000000002, | |
| "loss": 103.16549072265624, | |
| "original_ce_loss": 6.4472, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.006333333333333333, | |
| "grad_norm": 15.656424522399902, | |
| "learning_rate": 0.00015299999999999998, | |
| "loss": 102.62147216796875, | |
| "original_ce_loss": 6.4132, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.006666666666666667, | |
| "grad_norm": 18.646780014038086, | |
| "learning_rate": 0.0001615, | |
| "loss": 101.59368896484375, | |
| "original_ce_loss": 6.3489, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.007, | |
| "grad_norm": 39.721866607666016, | |
| "learning_rate": 0.00017, | |
| "loss": 100.74405517578126, | |
| "original_ce_loss": 6.2958, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.007333333333333333, | |
| "grad_norm": 16.912254333496094, | |
| "learning_rate": 0.00017849999999999997, | |
| "loss": 99.42055053710938, | |
| "original_ce_loss": 6.2131, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.007666666666666666, | |
| "grad_norm": 17.74278450012207, | |
| "learning_rate": 0.000187, | |
| "loss": 97.92646484375, | |
| "original_ce_loss": 6.1197, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.008, | |
| "grad_norm": 18.32769775390625, | |
| "learning_rate": 0.0001955, | |
| "loss": 96.8761962890625, | |
| "original_ce_loss": 6.0541, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.008333333333333333, | |
| "grad_norm": 16.8792781829834, | |
| "learning_rate": 0.00020399999999999997, | |
| "loss": 95.25513305664063, | |
| "original_ce_loss": 5.9528, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.008666666666666666, | |
| "grad_norm": 12.205955505371094, | |
| "learning_rate": 0.0002125, | |
| "loss": 93.572900390625, | |
| "original_ce_loss": 5.8476, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.009, | |
| "grad_norm": 13.931303024291992, | |
| "learning_rate": 0.000221, | |
| "loss": 91.66510009765625, | |
| "original_ce_loss": 5.7284, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.009333333333333334, | |
| "grad_norm": 26.242258071899414, | |
| "learning_rate": 0.0002295, | |
| "loss": 90.45693969726562, | |
| "original_ce_loss": 5.6529, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.009666666666666667, | |
| "grad_norm": 16.811609268188477, | |
| "learning_rate": 0.000238, | |
| "loss": 91.2010009765625, | |
| "original_ce_loss": 5.6994, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 25.731582641601562, | |
| "learning_rate": 0.00024565, | |
| "loss": 97.92516479492187, | |
| "original_ce_loss": 6.1197, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.010333333333333333, | |
| "grad_norm": 36.38114547729492, | |
| "learning_rate": 0.00025414999999999997, | |
| "loss": 98.45838623046875, | |
| "original_ce_loss": 6.153, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.010666666666666666, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00026179999999999997, | |
| "loss": 95.69706420898437, | |
| "original_ce_loss": 5.9804, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.011, | |
| "grad_norm": 51.799739837646484, | |
| "learning_rate": 0.00026944999999999996, | |
| "loss": 91.98704223632812, | |
| "original_ce_loss": 5.7485, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.011333333333333334, | |
| "grad_norm": 22.107471466064453, | |
| "learning_rate": 0.00027795, | |
| "loss": 85.44754028320312, | |
| "original_ce_loss": 5.3398, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.011666666666666667, | |
| "grad_norm": 30.49631118774414, | |
| "learning_rate": 0.00028645, | |
| "loss": 82.21802978515625, | |
| "original_ce_loss": 5.138, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.012, | |
| "grad_norm": 46.481483459472656, | |
| "learning_rate": 0.00029495, | |
| "loss": 81.45587768554688, | |
| "original_ce_loss": 5.0904, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.012333333333333333, | |
| "grad_norm": 35.435367584228516, | |
| "learning_rate": 0.00030345, | |
| "loss": 80.58915405273437, | |
| "original_ce_loss": 5.0362, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.012666666666666666, | |
| "grad_norm": 23.500009536743164, | |
| "learning_rate": 0.00031109999999999997, | |
| "loss": 79.21079711914062, | |
| "original_ce_loss": 4.95, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.013, | |
| "grad_norm": 36.313453674316406, | |
| "learning_rate": 0.00031959999999999996, | |
| "loss": 77.30345458984375, | |
| "original_ce_loss": 4.8308, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.013333333333333334, | |
| "grad_norm": 22.148427963256836, | |
| "learning_rate": 0.0003281, | |
| "loss": 75.77711181640625, | |
| "original_ce_loss": 4.7354, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.013666666666666667, | |
| "grad_norm": 25.187255859375, | |
| "learning_rate": 0.0003366, | |
| "loss": 73.7842041015625, | |
| "original_ce_loss": 4.6109, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.014, | |
| "grad_norm": 36.26912307739258, | |
| "learning_rate": 0.0003451, | |
| "loss": 71.966650390625, | |
| "original_ce_loss": 4.4973, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.014333333333333333, | |
| "grad_norm": 43.76127624511719, | |
| "learning_rate": 0.0003536, | |
| "loss": 70.83294677734375, | |
| "original_ce_loss": 4.4264, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.014666666666666666, | |
| "grad_norm": 26.379560470581055, | |
| "learning_rate": 0.00036209999999999997, | |
| "loss": 69.33712158203124, | |
| "original_ce_loss": 4.333, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.015, | |
| "grad_norm": 34.02827453613281, | |
| "learning_rate": 0.00037059999999999996, | |
| "loss": 67.307373046875, | |
| "original_ce_loss": 4.2061, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.015333333333333332, | |
| "grad_norm": 86.85484313964844, | |
| "learning_rate": 0.0003791, | |
| "loss": 65.93311767578125, | |
| "original_ce_loss": 4.1202, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.015666666666666666, | |
| "grad_norm": 39.64079666137695, | |
| "learning_rate": 0.0003876, | |
| "loss": 64.83297729492188, | |
| "original_ce_loss": 4.0515, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.016, | |
| "grad_norm": 27.540979385375977, | |
| "learning_rate": 0.0003961, | |
| "loss": 63.43541259765625, | |
| "original_ce_loss": 3.9641, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.01633333333333333, | |
| "grad_norm": 66.87540435791016, | |
| "learning_rate": 0.00040459999999999997, | |
| "loss": 62.5385986328125, | |
| "original_ce_loss": 3.9081, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.016666666666666666, | |
| "grad_norm": 43.895957946777344, | |
| "learning_rate": 0.00041309999999999996, | |
| "loss": 61.425079345703125, | |
| "original_ce_loss": 3.8385, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.017, | |
| "grad_norm": 29.485631942749023, | |
| "learning_rate": 0.00042159999999999995, | |
| "loss": 60.54182739257813, | |
| "original_ce_loss": 3.7833, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.017333333333333333, | |
| "grad_norm": 39.4123649597168, | |
| "learning_rate": 0.0004249999566202406, | |
| "loss": 59.02940673828125, | |
| "original_ce_loss": 3.6887, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.017666666666666667, | |
| "grad_norm": 29.954763412475586, | |
| "learning_rate": 0.0004249996915217756, | |
| "loss": 58.09852294921875, | |
| "original_ce_loss": 3.6306, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.018, | |
| "grad_norm": 23.342824935913086, | |
| "learning_rate": 0.00042499918542501206, | |
| "loss": 57.1560546875, | |
| "original_ce_loss": 3.5717, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.018333333333333333, | |
| "grad_norm": 28.458494186401367, | |
| "learning_rate": 0.00042499843833052406, | |
| "loss": 56.211798095703124, | |
| "original_ce_loss": 3.5127, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.018666666666666668, | |
| "grad_norm": 22.47995948791504, | |
| "learning_rate": 0.0004249974502391588, | |
| "loss": 55.34743041992188, | |
| "original_ce_loss": 3.4586, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.019, | |
| "grad_norm": 25.036964416503906, | |
| "learning_rate": 0.00042499622115203697, | |
| "loss": 54.5650390625, | |
| "original_ce_loss": 3.4097, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.019333333333333334, | |
| "grad_norm": 19.473806381225586, | |
| "learning_rate": 0.0004249947510705524, | |
| "loss": 54.05086059570313, | |
| "original_ce_loss": 3.3776, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.019666666666666666, | |
| "grad_norm": 31.730932235717773, | |
| "learning_rate": 0.0004249930399963724, | |
| "loss": 53.475189208984375, | |
| "original_ce_loss": 3.3416, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 161.93760681152344, | |
| "learning_rate": 0.0004249910879314376, | |
| "loss": 52.975347900390624, | |
| "original_ce_loss": 3.3104, | |
| "step": 600 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 30000, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 9223372036854775807, | |
| "save_steps": 200, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 4.42118754533376e+16, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |