Invalid JSON: Unexpected token 'N', ..."ad_norm": NaN,
"... is not valid JSON
| { | |
| "best_metric": 0.962536023054755, | |
| "best_model_checkpoint": "./roadwork-swin-finetuned/checkpoint-704", | |
| "epoch": 4.0, | |
| "eval_steps": 500, | |
| "global_step": 2816, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.07102272727272728, | |
| "grad_norm": 17.786890029907227, | |
| "learning_rate": 2.9488636363636363e-05, | |
| "loss": 0.416, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.14204545454545456, | |
| "grad_norm": 23.79692268371582, | |
| "learning_rate": 2.895596590909091e-05, | |
| "loss": 0.2916, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.21306818181818182, | |
| "grad_norm": 19.02972412109375, | |
| "learning_rate": 2.8433948863636362e-05, | |
| "loss": 0.2737, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2840909090909091, | |
| "grad_norm": 0.4028087556362152, | |
| "learning_rate": 2.790127840909091e-05, | |
| "loss": 0.375, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.35511363636363635, | |
| "grad_norm": 11.175421714782715, | |
| "learning_rate": 2.7368607954545453e-05, | |
| "loss": 0.3105, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.42613636363636365, | |
| "grad_norm": 14.542905807495117, | |
| "learning_rate": 2.68359375e-05, | |
| "loss": 0.2578, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.4971590909090909, | |
| "grad_norm": 0.4432804584503174, | |
| "learning_rate": 2.630326704545455e-05, | |
| "loss": 0.3166, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.5681818181818182, | |
| "grad_norm": 0.9627830386161804, | |
| "learning_rate": 2.5770596590909092e-05, | |
| "loss": 0.3289, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6392045454545454, | |
| "grad_norm": 5.985296249389648, | |
| "learning_rate": 2.523792613636364e-05, | |
| "loss": 0.2113, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7102272727272727, | |
| "grad_norm": 6.082040786743164, | |
| "learning_rate": 2.470525568181818e-05, | |
| "loss": 0.2488, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.78125, | |
| "grad_norm": 0.5786997079849243, | |
| "learning_rate": 2.4172585227272728e-05, | |
| "loss": 0.2465, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.8522727272727273, | |
| "grad_norm": 18.639421463012695, | |
| "learning_rate": 2.3639914772727272e-05, | |
| "loss": 0.2751, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.9232954545454546, | |
| "grad_norm": 15.852656364440918, | |
| "learning_rate": 2.310724431818182e-05, | |
| "loss": 0.2341, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.9943181818181818, | |
| "grad_norm": 31.554357528686523, | |
| "learning_rate": 2.2574573863636364e-05, | |
| "loss": 0.2625, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_accuracy": 0.9376996805111821, | |
| "eval_f1": 0.962536023054755, | |
| "eval_loss": 0.24441801011562347, | |
| "eval_precision": 0.9597701149425287, | |
| "eval_recall": 0.9653179190751445, | |
| "eval_runtime": 5.9751, | |
| "eval_samples_per_second": 104.769, | |
| "eval_steps_per_second": 13.222, | |
| "step": 704 | |
| }, | |
| { | |
| "epoch": 1.0653409090909092, | |
| "grad_norm": 0.20507046580314636, | |
| "learning_rate": 2.204190340909091e-05, | |
| "loss": 0.2122, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.1363636363636362, | |
| "grad_norm": 7.888465404510498, | |
| "learning_rate": 2.1509232954545455e-05, | |
| "loss": 0.1566, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.2073863636363638, | |
| "grad_norm": 1.0586321353912354, | |
| "learning_rate": 2.09765625e-05, | |
| "loss": 0.1512, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.2784090909090908, | |
| "grad_norm": 8.838766098022461, | |
| "learning_rate": 2.0443892045454547e-05, | |
| "loss": 0.2021, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.3494318181818181, | |
| "grad_norm": 0.0867004320025444, | |
| "learning_rate": 1.991122159090909e-05, | |
| "loss": 0.1948, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.4204545454545454, | |
| "grad_norm": NaN, | |
| "learning_rate": 1.9399857954545453e-05, | |
| "loss": 0.2295, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.4914772727272727, | |
| "grad_norm": 0.10102058947086334, | |
| "learning_rate": 1.88671875e-05, | |
| "loss": 0.1703, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.5625, | |
| "grad_norm": 0.09882424771785736, | |
| "learning_rate": 1.8334517045454548e-05, | |
| "loss": 0.24, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.6335227272727273, | |
| "grad_norm": 2.676793336868286, | |
| "learning_rate": 1.7801846590909092e-05, | |
| "loss": 0.1699, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.7045454545454546, | |
| "grad_norm": 1.2506362199783325, | |
| "learning_rate": 1.726917613636364e-05, | |
| "loss": 0.1621, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 1.7755681818181817, | |
| "grad_norm": 0.06482065469026566, | |
| "learning_rate": 1.673650568181818e-05, | |
| "loss": 0.1651, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 1.8465909090909092, | |
| "grad_norm": 0.20499753952026367, | |
| "learning_rate": 1.6203835227272727e-05, | |
| "loss": 0.2131, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 1.9176136363636362, | |
| "grad_norm": 41.24250030517578, | |
| "learning_rate": 1.567116477272727e-05, | |
| "loss": 0.223, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 1.9886363636363638, | |
| "grad_norm": 131.9430694580078, | |
| "learning_rate": 1.5138494318181819e-05, | |
| "loss": 0.132, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "eval_accuracy": 0.9265175718849841, | |
| "eval_f1": 0.9562737642585551, | |
| "eval_loss": 0.3970772624015808, | |
| "eval_precision": 0.9437148217636022, | |
| "eval_recall": 0.9691714836223507, | |
| "eval_runtime": 5.9491, | |
| "eval_samples_per_second": 105.226, | |
| "eval_steps_per_second": 13.279, | |
| "step": 1408 | |
| }, | |
| { | |
| "epoch": 2.059659090909091, | |
| "grad_norm": 30.692094802856445, | |
| "learning_rate": 1.4605823863636365e-05, | |
| "loss": 0.1146, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.1306818181818183, | |
| "grad_norm": 9.568278312683105, | |
| "learning_rate": 1.407315340909091e-05, | |
| "loss": 0.0766, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.2017045454545454, | |
| "grad_norm": 0.030713092535734177, | |
| "learning_rate": 1.3540482954545454e-05, | |
| "loss": 0.1294, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.2727272727272725, | |
| "grad_norm": 0.1593484729528427, | |
| "learning_rate": 1.30078125e-05, | |
| "loss": 0.1204, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.34375, | |
| "grad_norm": 0.012557649984955788, | |
| "learning_rate": 1.2475142045454546e-05, | |
| "loss": 0.051, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.4147727272727275, | |
| "grad_norm": 64.85875701904297, | |
| "learning_rate": 1.1942471590909092e-05, | |
| "loss": 0.178, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.4857954545454546, | |
| "grad_norm": 0.0735430121421814, | |
| "learning_rate": 1.1409801136363636e-05, | |
| "loss": 0.1308, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.5568181818181817, | |
| "grad_norm": 0.07014719396829605, | |
| "learning_rate": 1.0877130681818182e-05, | |
| "loss": 0.0831, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 2.627840909090909, | |
| "grad_norm": 19.661163330078125, | |
| "learning_rate": 1.0344460227272727e-05, | |
| "loss": 0.1815, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 2.6988636363636362, | |
| "grad_norm": 6.710304260253906, | |
| "learning_rate": 9.811789772727273e-06, | |
| "loss": 0.0773, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 2.7698863636363638, | |
| "grad_norm": 0.015027725137770176, | |
| "learning_rate": 9.279119318181819e-06, | |
| "loss": 0.0798, | |
| "step": 1950 | |
| }, | |
| { | |
| "epoch": 2.840909090909091, | |
| "grad_norm": 0.021411675959825516, | |
| "learning_rate": 8.746448863636365e-06, | |
| "loss": 0.1381, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 2.9119318181818183, | |
| "grad_norm": 36.07814025878906, | |
| "learning_rate": 8.21377840909091e-06, | |
| "loss": 0.0631, | |
| "step": 2050 | |
| }, | |
| { | |
| "epoch": 2.9829545454545454, | |
| "grad_norm": 0.362332284450531, | |
| "learning_rate": 7.681107954545454e-06, | |
| "loss": 0.1094, | |
| "step": 2100 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "eval_accuracy": 0.9361022364217252, | |
| "eval_f1": 0.9614643545279383, | |
| "eval_loss": 0.34437721967697144, | |
| "eval_precision": 0.9614643545279383, | |
| "eval_recall": 0.9614643545279383, | |
| "eval_runtime": 6.402, | |
| "eval_samples_per_second": 97.781, | |
| "eval_steps_per_second": 12.34, | |
| "step": 2112 | |
| }, | |
| { | |
| "epoch": 3.053977272727273, | |
| "grad_norm": 1.0269795656204224, | |
| "learning_rate": 7.1484375e-06, | |
| "loss": 0.0431, | |
| "step": 2150 | |
| }, | |
| { | |
| "epoch": 3.125, | |
| "grad_norm": 0.06955037266016006, | |
| "learning_rate": 6.615767045454546e-06, | |
| "loss": 0.044, | |
| "step": 2200 | |
| }, | |
| { | |
| "epoch": 3.196022727272727, | |
| "grad_norm": 0.2144497185945511, | |
| "learning_rate": 6.083096590909091e-06, | |
| "loss": 0.078, | |
| "step": 2250 | |
| }, | |
| { | |
| "epoch": 3.2670454545454546, | |
| "grad_norm": 0.8404433131217957, | |
| "learning_rate": 5.5504261363636365e-06, | |
| "loss": 0.0634, | |
| "step": 2300 | |
| }, | |
| { | |
| "epoch": 3.3380681818181817, | |
| "grad_norm": 0.018121657893061638, | |
| "learning_rate": 5.017755681818182e-06, | |
| "loss": 0.0263, | |
| "step": 2350 | |
| }, | |
| { | |
| "epoch": 3.409090909090909, | |
| "grad_norm": 0.9491384625434875, | |
| "learning_rate": 4.485085227272727e-06, | |
| "loss": 0.0913, | |
| "step": 2400 | |
| }, | |
| { | |
| "epoch": 3.4801136363636362, | |
| "grad_norm": 0.11287333071231842, | |
| "learning_rate": 3.952414772727273e-06, | |
| "loss": 0.0747, | |
| "step": 2450 | |
| }, | |
| { | |
| "epoch": 3.5511363636363638, | |
| "grad_norm": 10.997075080871582, | |
| "learning_rate": 3.4197443181818183e-06, | |
| "loss": 0.0654, | |
| "step": 2500 | |
| }, | |
| { | |
| "epoch": 3.622159090909091, | |
| "grad_norm": 0.04853343218564987, | |
| "learning_rate": 2.8870738636363636e-06, | |
| "loss": 0.0664, | |
| "step": 2550 | |
| }, | |
| { | |
| "epoch": 3.6931818181818183, | |
| "grad_norm": 29.09600067138672, | |
| "learning_rate": 2.354403409090909e-06, | |
| "loss": 0.0719, | |
| "step": 2600 | |
| }, | |
| { | |
| "epoch": 3.7642045454545454, | |
| "grad_norm": 0.372402161359787, | |
| "learning_rate": 1.8217329545454545e-06, | |
| "loss": 0.0713, | |
| "step": 2650 | |
| }, | |
| { | |
| "epoch": 3.8352272727272725, | |
| "grad_norm": 0.04578320309519768, | |
| "learning_rate": 1.2890625e-06, | |
| "loss": 0.0363, | |
| "step": 2700 | |
| }, | |
| { | |
| "epoch": 3.90625, | |
| "grad_norm": 0.3740798234939575, | |
| "learning_rate": 7.563920454545454e-07, | |
| "loss": 0.0614, | |
| "step": 2750 | |
| }, | |
| { | |
| "epoch": 3.9772727272727275, | |
| "grad_norm": 0.006558969616889954, | |
| "learning_rate": 2.2372159090909091e-07, | |
| "loss": 0.0539, | |
| "step": 2800 | |
| }, | |
| { | |
| "epoch": 4.0, | |
| "eval_accuracy": 0.9329073482428115, | |
| "eval_f1": 0.9593810444874274, | |
| "eval_loss": 0.41001200675964355, | |
| "eval_precision": 0.9631067961165048, | |
| "eval_recall": 0.9556840077071291, | |
| "eval_runtime": 5.9591, | |
| "eval_samples_per_second": 105.049, | |
| "eval_steps_per_second": 13.257, | |
| "step": 2816 | |
| } | |
| ], | |
| "logging_steps": 50, | |
| "max_steps": 2816, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 4, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 1.76277500163072e+18, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |