Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN,
"... is not valid JSON
| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.481203007518797, | |
| "eval_steps": 500, | |
| "global_step": 600, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.041483017889551464, | |
| "grad_norm": 203.76828002929688, | |
| "learning_rate": 1.4875e-05, | |
| "loss": 288.5525390625, | |
| "original_ce_loss": 9.0166, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.08296603577910293, | |
| "grad_norm": 184.2494659423828, | |
| "learning_rate": 3.6125000000000004e-05, | |
| "loss": 287.5813232421875, | |
| "original_ce_loss": 8.9862, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.1244490536686544, | |
| "grad_norm": 241.391845703125, | |
| "learning_rate": 5.7375e-05, | |
| "loss": 284.6296875, | |
| "original_ce_loss": 8.894, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.16593207155820586, | |
| "grad_norm": 327.870361328125, | |
| "learning_rate": 7.8625e-05, | |
| "loss": 273.14404296875, | |
| "original_ce_loss": 8.5351, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.20741508944775733, | |
| "grad_norm": 80.51351165771484, | |
| "learning_rate": 9.775e-05, | |
| "loss": 243.771630859375, | |
| "original_ce_loss": 7.6172, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.2488981073373088, | |
| "grad_norm": 61.04130554199219, | |
| "learning_rate": 0.00011687500000000001, | |
| "loss": 229.8462646484375, | |
| "original_ce_loss": 7.182, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.29038112522686027, | |
| "grad_norm": 60.73427200317383, | |
| "learning_rate": 0.000129625, | |
| "loss": 221.95341796875, | |
| "original_ce_loss": 6.9354, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.3318641431164117, | |
| "grad_norm": 51.37192916870117, | |
| "learning_rate": 0.000150875, | |
| "loss": 214.37080078125, | |
| "original_ce_loss": 6.6984, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.37334716100596316, | |
| "grad_norm": 22.531389236450195, | |
| "learning_rate": 0.000172125, | |
| "loss": 205.998388671875, | |
| "original_ce_loss": 6.4368, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.41483017889551466, | |
| "grad_norm": 16.882768630981445, | |
| "learning_rate": 0.000193375, | |
| "loss": 201.5771484375, | |
| "original_ce_loss": 6.2986, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.4563131967850661, | |
| "grad_norm": 32.13435363769531, | |
| "learning_rate": 0.000214625, | |
| "loss": 199.65380859375, | |
| "original_ce_loss": 6.2385, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.4977962146746176, | |
| "grad_norm": 22.2371883392334, | |
| "learning_rate": 0.000235875, | |
| "loss": 198.15953369140624, | |
| "original_ce_loss": 6.1918, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.5392792325641691, | |
| "grad_norm": 23.51988983154297, | |
| "learning_rate": 0.000257125, | |
| "loss": 195.21707763671876, | |
| "original_ce_loss": 6.0999, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.5807622504537205, | |
| "grad_norm": 21.056625366210938, | |
| "learning_rate": 0.000278375, | |
| "loss": 192.00562744140626, | |
| "original_ce_loss": 5.9995, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.622245268343272, | |
| "grad_norm": 27.031875610351562, | |
| "learning_rate": 0.000299625, | |
| "loss": 189.01895751953126, | |
| "original_ce_loss": 5.9062, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.6637282862328234, | |
| "grad_norm": 18.687732696533203, | |
| "learning_rate": 0.000320875, | |
| "loss": 185.54974365234375, | |
| "original_ce_loss": 5.7978, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.7052113041223749, | |
| "grad_norm": 34.62672805786133, | |
| "learning_rate": 0.000342125, | |
| "loss": 181.4411865234375, | |
| "original_ce_loss": 5.6694, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.7466943220119263, | |
| "grad_norm": 22.933151245117188, | |
| "learning_rate": 0.000363375, | |
| "loss": 176.920361328125, | |
| "original_ce_loss": 5.5281, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.7881773399014779, | |
| "grad_norm": 40.35688018798828, | |
| "learning_rate": 0.000384625, | |
| "loss": 171.9557861328125, | |
| "original_ce_loss": 5.373, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.8296603577910293, | |
| "grad_norm": 36.39991760253906, | |
| "learning_rate": 0.000405875, | |
| "loss": 167.02354736328124, | |
| "original_ce_loss": 5.2188, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.8711433756805808, | |
| "grad_norm": 50.5265998840332, | |
| "learning_rate": 0.00042440580690100075, | |
| "loss": 162.07708740234375, | |
| "original_ce_loss": 5.0643, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.9126263935701322, | |
| "grad_norm": 37.708221435546875, | |
| "learning_rate": 0.00035703670677632033, | |
| "loss": 157.2227294921875, | |
| "original_ce_loss": 4.9126, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.9541094114596836, | |
| "grad_norm": 35.469329833984375, | |
| "learning_rate": 0.0002125, | |
| "loss": 153.1784423828125, | |
| "original_ce_loss": 4.7862, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.9955924293492352, | |
| "grad_norm": 16.84328842163086, | |
| "learning_rate": 6.796329322367959e-05, | |
| "loss": 149.57703857421876, | |
| "original_ce_loss": 4.6736, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 1.0331864143116412, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00040351662824681024, | |
| "loss": 149.98463439941406, | |
| "original_ce_loss": 4.6864, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.0746694322011927, | |
| "grad_norm": 64.90169525146484, | |
| "learning_rate": 0.00039331657240756313, | |
| "loss": 145.989599609375, | |
| "original_ce_loss": 4.5615, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 1.116152450090744, | |
| "grad_norm": 42.50829315185547, | |
| "learning_rate": 0.00037988794433001515, | |
| "loss": 141.3390625, | |
| "original_ce_loss": 4.4162, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 1.1576354679802956, | |
| "grad_norm": 38.33572769165039, | |
| "learning_rate": 0.0003644131338923845, | |
| "loss": 138.03270263671874, | |
| "original_ce_loss": 4.3129, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 1.199118485869847, | |
| "grad_norm": 40.65241241455078, | |
| "learning_rate": 0.00034708130814771, | |
| "loss": 135.1212158203125, | |
| "original_ce_loss": 4.2219, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 1.2406015037593985, | |
| "grad_norm": 69.57759857177734, | |
| "learning_rate": 0.0003281043346605806, | |
| "loss": 132.01983642578125, | |
| "original_ce_loss": 4.125, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.28208452164895, | |
| "grad_norm": 47.51698684692383, | |
| "learning_rate": 0.00030771419159736043, | |
| "loss": 128.94189453125, | |
| "original_ce_loss": 4.0288, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.3235675395385014, | |
| "grad_norm": 46.686241149902344, | |
| "learning_rate": 0.0002861601319805566, | |
| "loss": 126.20169677734376, | |
| "original_ce_loss": 3.9432, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.365050557428053, | |
| "grad_norm": 39.812068939208984, | |
| "learning_rate": 0.00026370563677203054, | |
| "loss": 123.58370361328124, | |
| "original_ce_loss": 3.8614, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.4065335753176043, | |
| "grad_norm": 34.92276382446289, | |
| "learning_rate": 0.0002406251940311473, | |
| "loss": 121.514599609375, | |
| "original_ce_loss": 3.7967, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.4480165932071558, | |
| "grad_norm": 51.240840911865234, | |
| "learning_rate": 0.0002172009435200522, | |
| "loss": 118.99783935546876, | |
| "original_ce_loss": 3.7181, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.4894996110967074, | |
| "grad_norm": 42.60438537597656, | |
| "learning_rate": 0.00019371922777306473, | |
| "loss": 117.68125, | |
| "original_ce_loss": 3.6769, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.5309826289862587, | |
| "grad_norm": 32.416114807128906, | |
| "learning_rate": 0.00017046709179057757, | |
| "loss": 115.90443115234375, | |
| "original_ce_loss": 3.6214, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.5724656468758103, | |
| "grad_norm": 31.31523895263672, | |
| "learning_rate": 0.0001477287741458766, | |
| "loss": 114.24423828125, | |
| "original_ce_loss": 3.5695, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.6139486647653616, | |
| "grad_norm": 29.63783836364746, | |
| "learning_rate": 0.00012578223239826367, | |
| "loss": 113.076904296875, | |
| "original_ce_loss": 3.533, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.6554316826549131, | |
| "grad_norm": 21.96636962890625, | |
| "learning_rate": 0.0001048957452865007, | |
| "loss": 111.66766357421875, | |
| "original_ce_loss": 3.489, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.6969147005444647, | |
| "grad_norm": 72.03860473632812, | |
| "learning_rate": 0.00029176382908428956, | |
| "loss": 111.707080078125, | |
| "original_ce_loss": 3.4902, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 1.738397718434016, | |
| "grad_norm": 56.377784729003906, | |
| "learning_rate": 0.0002798536820919388, | |
| "loss": 110.67801513671876, | |
| "original_ce_loss": 3.4581, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 1.7798807363235676, | |
| "grad_norm": 40.06888198852539, | |
| "learning_rate": 0.0002677033420177621, | |
| "loss": 108.81322021484375, | |
| "original_ce_loss": 3.3998, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 1.821363754213119, | |
| "grad_norm": 54.35465621948242, | |
| "learning_rate": 0.0002553561387516336, | |
| "loss": 107.58404541015625, | |
| "original_ce_loss": 3.3614, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 1.8628467721026705, | |
| "grad_norm": 43.010047912597656, | |
| "learning_rate": 0.00024285610422636377, | |
| "loss": 106.3376953125, | |
| "original_ce_loss": 3.3225, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.904329789992222, | |
| "grad_norm": 37.77562713623047, | |
| "learning_rate": 0.00023024781539338732, | |
| "loss": 105.1876708984375, | |
| "original_ce_loss": 3.2865, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 1.9458128078817734, | |
| "grad_norm": 39.30547332763672, | |
| "learning_rate": 0.00021757623525483388, | |
| "loss": 104.48690185546874, | |
| "original_ce_loss": 3.2646, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.987295825771325, | |
| "grad_norm": 38.75474548339844, | |
| "learning_rate": 0.00020615504546354557, | |
| "loss": 103.30198974609375, | |
| "original_ce_loss": 3.2276, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 2.0248898107337308, | |
| "grad_norm": 68.0500259399414, | |
| "learning_rate": 0.00019348776342655851, | |
| "loss": 93.47062377929687, | |
| "original_ce_loss": 3.2225, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 2.0663728286232823, | |
| "grad_norm": 33.5322380065918, | |
| "learning_rate": 0.00018088828180601698, | |
| "loss": 102.30059814453125, | |
| "original_ce_loss": 3.1963, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.107855846512834, | |
| "grad_norm": 43.00708770751953, | |
| "learning_rate": 0.00016840153219624304, | |
| "loss": 101.05491943359375, | |
| "original_ce_loss": 3.1574, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 2.1493388644023854, | |
| "grad_norm": 32.75222396850586, | |
| "learning_rate": 0.00015607204417271948, | |
| "loss": 100.06157836914062, | |
| "original_ce_loss": 3.1263, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 2.190821882291937, | |
| "grad_norm": 34.5550537109375, | |
| "learning_rate": 0.0001439437864931107, | |
| "loss": 99.03675537109375, | |
| "original_ce_loss": 3.0943, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 2.232304900181488, | |
| "grad_norm": 33.61601257324219, | |
| "learning_rate": 0.00013206001029824168, | |
| "loss": 98.9501953125, | |
| "original_ce_loss": 3.0916, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 2.2737879180710396, | |
| "grad_norm": 35.476318359375, | |
| "learning_rate": 0.00012046309487220386, | |
| "loss": 98.14458618164062, | |
| "original_ce_loss": 3.0664, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 2.315270935960591, | |
| "grad_norm": 27.919132232666016, | |
| "learning_rate": 0.00010919439651162934, | |
| "loss": 97.50660400390625, | |
| "original_ce_loss": 3.0465, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 2.3567539538501427, | |
| "grad_norm": 27.541818618774414, | |
| "learning_rate": 9.829410104308781e-05, | |
| "loss": 96.76535034179688, | |
| "original_ce_loss": 3.0233, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 2.398236971739694, | |
| "grad_norm": 25.62722396850586, | |
| "learning_rate": 8.780108051455205e-05, | |
| "loss": 96.71749877929688, | |
| "original_ce_loss": 3.0218, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 2.4397199896292454, | |
| "grad_norm": 31.261812210083008, | |
| "learning_rate": 7.775275457198821e-05, | |
| "loss": 95.91024169921874, | |
| "original_ce_loss": 2.9966, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 2.481203007518797, | |
| "grad_norm": 25.36027717590332, | |
| "learning_rate": 6.818495701542595e-05, | |
| "loss": 95.57075805664063, | |
| "original_ce_loss": 2.986, | |
| "step": 600 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 726, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 200, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 4.867238337530266e+16, | |
| "train_batch_size": 64, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |