Invalid JSON:
Unexpected token 'N', ..."ad_norm": NaN,
"... is not valid JSON
| { | |
| "best_metric": 1.6491321325302124, | |
| "best_model_checkpoint": "/data/user_data/gonilude/python_and_text_pythia_160m/checkpoint-150", | |
| "epoch": 3.0, | |
| "eval_steps": 50, | |
| "global_step": 237, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0, | |
| "eval_accuracy": 0.19718309859154928, | |
| "eval_loss": 9.814414978027344, | |
| "eval_runtime": 0.3368, | |
| "eval_samples_per_second": 210.811, | |
| "eval_steps_per_second": 26.723, | |
| "num_input_tokens_seen": 0, | |
| "step": 0 | |
| }, | |
| { | |
| "epoch": 0.012658227848101266, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.0, | |
| "loss": 9.425, | |
| "num_input_tokens_seen": 8192, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06329113924050633, | |
| "grad_norm": Infinity, | |
| "learning_rate": 0.0, | |
| "loss": 7.9382, | |
| "num_input_tokens_seen": 40960, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.12658227848101267, | |
| "grad_norm": Infinity, | |
| "learning_rate": 5e-06, | |
| "loss": 8.0612, | |
| "num_input_tokens_seen": 81920, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.189873417721519, | |
| "grad_norm": 377.5516662597656, | |
| "learning_rate": 1.7500000000000002e-05, | |
| "loss": 6.289, | |
| "num_input_tokens_seen": 122880, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.25316455696202533, | |
| "grad_norm": 519.905517578125, | |
| "learning_rate": 1.9984947460216708e-05, | |
| "loss": 3.6305, | |
| "num_input_tokens_seen": 163840, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.31645569620253167, | |
| "grad_norm": 137.5762939453125, | |
| "learning_rate": 1.9923874174311394e-05, | |
| "loss": 2.816, | |
| "num_input_tokens_seen": 204800, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.379746835443038, | |
| "grad_norm": 152.59420776367188, | |
| "learning_rate": 1.9816126380411478e-05, | |
| "loss": 1.8783, | |
| "num_input_tokens_seen": 245760, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.4430379746835443, | |
| "grad_norm": 184.83087158203125, | |
| "learning_rate": 1.9662210843836574e-05, | |
| "loss": 2.1072, | |
| "num_input_tokens_seen": 286720, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.5063291139240507, | |
| "grad_norm": 186.87013244628906, | |
| "learning_rate": 1.946285146855968e-05, | |
| "loss": 1.958, | |
| "num_input_tokens_seen": 327680, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.569620253164557, | |
| "grad_norm": 170.01846313476562, | |
| "learning_rate": 1.921898589250242e-05, | |
| "loss": 2.15, | |
| "num_input_tokens_seen": 368640, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.6329113924050633, | |
| "grad_norm": 156.2119598388672, | |
| "learning_rate": 1.8931761077585037e-05, | |
| "loss": 2.0211, | |
| "num_input_tokens_seen": 409600, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.6329113924050633, | |
| "eval_accuracy": 0.1267605633802817, | |
| "eval_loss": 1.9534826278686523, | |
| "eval_runtime": 0.2174, | |
| "eval_samples_per_second": 326.596, | |
| "eval_steps_per_second": 41.4, | |
| "num_input_tokens_seen": 409600, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.6962025316455697, | |
| "grad_norm": 125.18607330322266, | |
| "learning_rate": 1.860252791527236e-05, | |
| "loss": 1.8652, | |
| "num_input_tokens_seen": 450560, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.759493670886076, | |
| "grad_norm": 131.51097106933594, | |
| "learning_rate": 1.8232834872987147e-05, | |
| "loss": 1.6875, | |
| "num_input_tokens_seen": 491520, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.8227848101265823, | |
| "grad_norm": 132.37973022460938, | |
| "learning_rate": 1.782442071127338e-05, | |
| "loss": 1.8354, | |
| "num_input_tokens_seen": 532480, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.8860759493670886, | |
| "grad_norm": 158.42999267578125, | |
| "learning_rate": 1.7379206305962525e-05, | |
| "loss": 1.8209, | |
| "num_input_tokens_seen": 573440, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.9493670886075949, | |
| "grad_norm": 67.9870834350586, | |
| "learning_rate": 1.6899285613805246e-05, | |
| "loss": 1.7624, | |
| "num_input_tokens_seen": 614400, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 1.0126582278481013, | |
| "grad_norm": 178.30116271972656, | |
| "learning_rate": 1.6386915824059427e-05, | |
| "loss": 1.7832, | |
| "num_input_tokens_seen": 655360, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.0759493670886076, | |
| "grad_norm": 185.28076171875, | |
| "learning_rate": 1.5844506742354163e-05, | |
| "loss": 2.1439, | |
| "num_input_tokens_seen": 696320, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 1.139240506329114, | |
| "grad_norm": 157.6683807373047, | |
| "learning_rate": 1.5274609456760073e-05, | |
| "loss": 1.8426, | |
| "num_input_tokens_seen": 737280, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.2025316455696202, | |
| "grad_norm": 107.63882446289062, | |
| "learning_rate": 1.4679904339372301e-05, | |
| "loss": 1.7307, | |
| "num_input_tokens_seen": 778240, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 1.2658227848101267, | |
| "grad_norm": 126.8455581665039, | |
| "learning_rate": 1.4063188439837831e-05, | |
| "loss": 1.916, | |
| "num_input_tokens_seen": 819200, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.2658227848101267, | |
| "eval_accuracy": 0.19718309859154928, | |
| "eval_loss": 1.6971796751022339, | |
| "eval_runtime": 0.219, | |
| "eval_samples_per_second": 324.264, | |
| "eval_steps_per_second": 41.104, | |
| "num_input_tokens_seen": 819200, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.3291139240506329, | |
| "grad_norm": 104.72110748291016, | |
| "learning_rate": 1.3427362330118542e-05, | |
| "loss": 1.7162, | |
| "num_input_tokens_seen": 860160, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 1.3924050632911391, | |
| "grad_norm": 93.7473373413086, | |
| "learning_rate": 1.2775416462362458e-05, | |
| "loss": 1.6155, | |
| "num_input_tokens_seen": 901120, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.4556962025316456, | |
| "grad_norm": 83.0439682006836, | |
| "learning_rate": 1.2110417104045575e-05, | |
| "loss": 1.7444, | |
| "num_input_tokens_seen": 942080, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 1.518987341772152, | |
| "grad_norm": 88.41893768310547, | |
| "learning_rate": 1.1435491916534919e-05, | |
| "loss": 1.6476, | |
| "num_input_tokens_seen": 983040, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.5822784810126582, | |
| "grad_norm": 49.397403717041016, | |
| "learning_rate": 1.0753815244900459e-05, | |
| "loss": 1.5816, | |
| "num_input_tokens_seen": 1024000, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.6455696202531644, | |
| "grad_norm": 87.76904296875, | |
| "learning_rate": 1.0068593188161698e-05, | |
| "loss": 2.0688, | |
| "num_input_tokens_seen": 1064960, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 1.7088607594936709, | |
| "grad_norm": 60.90505599975586, | |
| "learning_rate": 9.383048520187344e-06, | |
| "loss": 1.6899, | |
| "num_input_tokens_seen": 1105920, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 1.7721518987341773, | |
| "grad_norm": 80.01156616210938, | |
| "learning_rate": 8.700405532168921e-06, | |
| "loss": 1.6115, | |
| "num_input_tokens_seen": 1146880, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.8354430379746836, | |
| "grad_norm": 92.96195983886719, | |
| "learning_rate": 8.023874867958027e-06, | |
| "loss": 1.5858, | |
| "num_input_tokens_seen": 1187840, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 1.8987341772151898, | |
| "grad_norm": 149.9720001220703, | |
| "learning_rate": 7.3566384235904855e-06, | |
| "loss": 1.6616, | |
| "num_input_tokens_seen": 1228800, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.8987341772151898, | |
| "eval_accuracy": 0.19718309859154928, | |
| "eval_loss": 1.6491321325302124, | |
| "eval_runtime": 0.1676, | |
| "eval_samples_per_second": 423.615, | |
| "eval_steps_per_second": 53.698, | |
| "num_input_tokens_seen": 1228800, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.9620253164556962, | |
| "grad_norm": 57.700984954833984, | |
| "learning_rate": 6.7018343820188324e-06, | |
| "loss": 1.5952, | |
| "num_input_tokens_seen": 1269760, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 2.0253164556962027, | |
| "grad_norm": 66.33654022216797, | |
| "learning_rate": 6.0625424534385425e-06, | |
| "loss": 1.5148, | |
| "num_input_tokens_seen": 1310720, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 2.088607594936709, | |
| "grad_norm": 91.3177490234375, | |
| "learning_rate": 5.441769390626537e-06, | |
| "loss": 1.6566, | |
| "num_input_tokens_seen": 1351680, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 2.151898734177215, | |
| "grad_norm": 91.95642852783203, | |
| "learning_rate": 4.842434847417001e-06, | |
| "loss": 1.6347, | |
| "num_input_tokens_seen": 1392640, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 2.2151898734177213, | |
| "grad_norm": 118.14971160888672, | |
| "learning_rate": 4.267357646825746e-06, | |
| "loss": 1.5415, | |
| "num_input_tokens_seen": 1433600, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 2.278481012658228, | |
| "grad_norm": 154.98838806152344, | |
| "learning_rate": 3.719242523407539e-06, | |
| "loss": 1.6275, | |
| "num_input_tokens_seen": 1474560, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 2.3417721518987342, | |
| "grad_norm": 97.98387908935547, | |
| "learning_rate": 3.2006674022005857e-06, | |
| "loss": 1.6091, | |
| "num_input_tokens_seen": 1515520, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 2.4050632911392404, | |
| "grad_norm": 51.0837287902832, | |
| "learning_rate": 2.714071274088438e-06, | |
| "loss": 1.5801, | |
| "num_input_tokens_seen": 1556480, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 2.4683544303797467, | |
| "grad_norm": 158.22381591796875, | |
| "learning_rate": 2.2617427246045976e-06, | |
| "loss": 1.6031, | |
| "num_input_tokens_seen": 1597440, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 2.5316455696202533, | |
| "grad_norm": 66.8087387084961, | |
| "learning_rate": 1.8458091701318504e-06, | |
| "loss": 1.565, | |
| "num_input_tokens_seen": 1638400, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 2.5316455696202533, | |
| "eval_accuracy": 0.14084507042253522, | |
| "eval_loss": 1.6663732528686523, | |
| "eval_runtime": 0.2173, | |
| "eval_samples_per_second": 326.688, | |
| "eval_steps_per_second": 41.411, | |
| "num_input_tokens_seen": 1638400, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 2.5949367088607596, | |
| "grad_norm": 82.30987548828125, | |
| "learning_rate": 1.4682268521211075e-06, | |
| "loss": 1.5668, | |
| "num_input_tokens_seen": 1679360, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 2.6582278481012658, | |
| "grad_norm": 54.035919189453125, | |
| "learning_rate": 1.130771636389596e-06, | |
| "loss": 1.5857, | |
| "num_input_tokens_seen": 1720320, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 2.721518987341772, | |
| "grad_norm": 80.6180191040039, | |
| "learning_rate": 8.350306607715774e-07, | |
| "loss": 1.593, | |
| "num_input_tokens_seen": 1761280, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 2.7848101265822782, | |
| "grad_norm": 38.75018310546875, | |
| "learning_rate": 5.823948704048443e-07, | |
| "loss": 1.5226, | |
| "num_input_tokens_seen": 1802240, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 2.848101265822785, | |
| "grad_norm": 99.82989501953125, | |
| "learning_rate": 3.7405247576144055e-07, | |
| "loss": 1.6135, | |
| "num_input_tokens_seen": 1843200, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 2.911392405063291, | |
| "grad_norm": 74.42279815673828, | |
| "learning_rate": 2.1098336419116628e-07, | |
| "loss": 1.5738, | |
| "num_input_tokens_seen": 1884160, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 2.9746835443037973, | |
| "grad_norm": 124.18939208984375, | |
| "learning_rate": 9.395449126177291e-08, | |
| "loss": 1.5493, | |
| "num_input_tokens_seen": 1925120, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "num_input_tokens_seen": 1941504, | |
| "step": 237, | |
| "total_flos": 990864117596160.0, | |
| "train_loss": 2.1522638506024196, | |
| "train_runtime": 52.4434, | |
| "train_samples_per_second": 36.096, | |
| "train_steps_per_second": 4.519 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 237, | |
| "num_input_tokens_seen": 1941504, | |
| "num_train_epochs": 3, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 990864117596160.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |