MsIssuesBERT / run-4 /checkpoint-4000 /trainer_state.json
YagiASAFAS's picture
Training in progress, epoch 1
4b17557 verified
Invalid JSON: Unexpected token 'N', ..."al_loss": NaN, "... is not valid JSON
{
"best_global_step": 4000,
"best_metric": 0.8135900008491362,
"best_model_checkpoint": "./results/run-4/checkpoint-4000",
"epoch": 4.0,
"eval_steps": 500,
"global_step": 4000,
"is_hyper_param_search": true,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.1,
"grad_norm": 3.002380609512329,
"learning_rate": 4.572943108695439e-06,
"loss": 0.296,
"step": 100
},
{
"epoch": 0.2,
"grad_norm": 0.9367781281471252,
"learning_rate": 9.192077561923155e-06,
"loss": 1.0274,
"step": 200
},
{
"epoch": 0.3,
"grad_norm": 0.7274576425552368,
"learning_rate": 1.3811212015150872e-05,
"loss": 0.0,
"step": 300
},
{
"epoch": 0.4,
"grad_norm": 44.297061920166016,
"learning_rate": 1.8430346468378586e-05,
"loss": 0.0,
"step": 400
},
{
"epoch": 0.5,
"grad_norm": 1.0151411294937134,
"learning_rate": 2.3049480921606306e-05,
"loss": 0.0,
"step": 500
},
{
"epoch": 0.6,
"grad_norm": 1.208032488822937,
"learning_rate": 2.766861537483402e-05,
"loss": 0.0,
"step": 600
},
{
"epoch": 0.7,
"grad_norm": 6.984821319580078,
"learning_rate": 3.2287749828061733e-05,
"loss": 0.0342,
"step": 700
},
{
"epoch": 0.8,
"grad_norm": 1.5860588550567627,
"learning_rate": 3.690688428128945e-05,
"loss": 0.0488,
"step": 800
},
{
"epoch": 0.9,
"grad_norm": 0.7841507196426392,
"learning_rate": 4.152601873451717e-05,
"loss": 0.0314,
"step": 900
},
{
"epoch": 1.0,
"grad_norm": 0.9344207644462585,
"learning_rate": 4.414230747586468e-05,
"loss": 0.0356,
"step": 1000
},
{
"epoch": 1.0,
"eval_economic_inequality_accuracy": 0.7783375314861462,
"eval_economic_inequality_f1": 0.6829252350399464,
"eval_economic_policy_benefits_accuracy": 0.8215488215488216,
"eval_economic_policy_benefits_f1": 0.752833443442648,
"eval_ethnic_boundaries_accuracy": 0.9493464052287581,
"eval_ethnic_boundaries_f1": 0.9305506350938352,
"eval_language_policy_accuracy": 0.6764705882352942,
"eval_language_policy_f1": 0.555672268907563,
"eval_loss": NaN,
"eval_mother_tongue_education_accuracy": 0.7777777777777778,
"eval_mother_tongue_education_f1": 0.6805555555555556,
"eval_overall_accuracy": 0.8090908256773385,
"eval_overall_f1": 0.7331392486819855,
"eval_religion_ethnic_identity_accuracy": 0.851063829787234,
"eval_religion_ethnic_identity_f1": 0.7962983540523647,
"eval_runtime": 4.4716,
"eval_samples_per_second": 894.535,
"eval_steps_per_second": 55.908,
"step": 1000
},
{
"epoch": 1.1,
"grad_norm": 0.8767189383506775,
"learning_rate": 4.303902560943467e-05,
"loss": 0.3117,
"step": 1100
},
{
"epoch": 1.2,
"grad_norm": 32.66496658325195,
"learning_rate": 4.193574374300466e-05,
"loss": 0.0098,
"step": 1200
},
{
"epoch": 1.3,
"grad_norm": 6.170604705810547,
"learning_rate": 4.083246187657465e-05,
"loss": 0.0,
"step": 1300
},
{
"epoch": 1.4,
"grad_norm": 0.40170523524284363,
"learning_rate": 3.972918001014464e-05,
"loss": 0.2259,
"step": 1400
},
{
"epoch": 1.5,
"grad_norm": 3.7784886360168457,
"learning_rate": 3.862589814371463e-05,
"loss": 0.0,
"step": 1500
},
{
"epoch": 1.6,
"grad_norm": 1.0622837543487549,
"learning_rate": 3.752261627728462e-05,
"loss": 0.0,
"step": 1600
},
{
"epoch": 1.7,
"grad_norm": 0.3284582793712616,
"learning_rate": 3.641933441085462e-05,
"loss": 0.0012,
"step": 1700
},
{
"epoch": 1.8,
"grad_norm": 3.2630698680877686,
"learning_rate": 3.53160525444246e-05,
"loss": 0.0045,
"step": 1800
},
{
"epoch": 1.9,
"grad_norm": 0.4251757860183716,
"learning_rate": 3.421277067799459e-05,
"loss": 0.1053,
"step": 1900
},
{
"epoch": 2.0,
"grad_norm": 2.5576839447021484,
"learning_rate": 3.310948881156459e-05,
"loss": 0.0826,
"step": 2000
},
{
"epoch": 2.0,
"eval_economic_inequality_accuracy": 0.7984886649874056,
"eval_economic_inequality_f1": 0.792653055614301,
"eval_economic_policy_benefits_accuracy": 0.8215488215488216,
"eval_economic_policy_benefits_f1": 0.8093298577169545,
"eval_ethnic_boundaries_accuracy": 0.9444444444444444,
"eval_ethnic_boundaries_f1": 0.9370640143333165,
"eval_language_policy_accuracy": 0.5882352941176471,
"eval_language_policy_f1": 0.5458823529411765,
"eval_loss": NaN,
"eval_mother_tongue_education_accuracy": 0.8888888888888888,
"eval_mother_tongue_education_f1": 0.837037037037037,
"eval_overall_accuracy": 0.8148002453034774,
"eval_overall_f1": 0.7921517057928192,
"eval_religion_ethnic_identity_accuracy": 0.8471953578336557,
"eval_religion_ethnic_identity_f1": 0.8309439171141297,
"eval_runtime": 4.6859,
"eval_samples_per_second": 853.616,
"eval_steps_per_second": 53.351,
"step": 2000
},
{
"epoch": 2.1,
"grad_norm": 210.1742706298828,
"learning_rate": 3.200620694513457e-05,
"loss": 1.2274,
"step": 2100
},
{
"epoch": 2.2,
"grad_norm": 3.0127012729644775,
"learning_rate": 3.090292507870457e-05,
"loss": 0.0,
"step": 2200
},
{
"epoch": 2.3,
"grad_norm": 0.38292792439460754,
"learning_rate": 2.9799643212274557e-05,
"loss": 0.0351,
"step": 2300
},
{
"epoch": 2.4,
"grad_norm": 2.8801395893096924,
"learning_rate": 2.8696361345844544e-05,
"loss": 0.0026,
"step": 2400
},
{
"epoch": 2.5,
"grad_norm": 1.5863370895385742,
"learning_rate": 2.7593079479414537e-05,
"loss": 0.0,
"step": 2500
},
{
"epoch": 2.6,
"grad_norm": 0.23432491719722748,
"learning_rate": 2.6489797612984527e-05,
"loss": 0.1192,
"step": 2600
},
{
"epoch": 2.7,
"grad_norm": 1.943647027015686,
"learning_rate": 2.5386515746554517e-05,
"loss": 0.0195,
"step": 2700
},
{
"epoch": 2.8,
"grad_norm": 0.5006923079490662,
"learning_rate": 2.4283233880124507e-05,
"loss": 0.0,
"step": 2800
},
{
"epoch": 2.9,
"grad_norm": 0.26087385416030884,
"learning_rate": 2.31799520136945e-05,
"loss": 0.0,
"step": 2900
},
{
"epoch": 3.0,
"grad_norm": 1.207767367362976,
"learning_rate": 2.207667014726449e-05,
"loss": 0.1565,
"step": 3000
},
{
"epoch": 3.0,
"eval_economic_inequality_accuracy": 0.8148614609571788,
"eval_economic_inequality_f1": 0.7997421102149133,
"eval_economic_policy_benefits_accuracy": 0.8518518518518519,
"eval_economic_policy_benefits_f1": 0.8175032474456659,
"eval_ethnic_boundaries_accuracy": 0.9477124183006536,
"eval_ethnic_boundaries_f1": 0.9334997758629278,
"eval_language_policy_accuracy": 0.6764705882352942,
"eval_language_policy_f1": 0.5869281045751634,
"eval_loss": NaN,
"eval_mother_tongue_education_accuracy": 0.8888888888888888,
"eval_mother_tongue_education_f1": 0.837037037037037,
"eval_overall_accuracy": 0.8371853490189908,
"eval_overall_f1": 0.7988061056554884,
"eval_religion_ethnic_identity_accuracy": 0.8433268858800773,
"eval_religion_ethnic_identity_f1": 0.8181263587972226,
"eval_runtime": 4.4894,
"eval_samples_per_second": 890.983,
"eval_steps_per_second": 55.686,
"step": 3000
},
{
"epoch": 3.1,
"grad_norm": 0.10033319890499115,
"learning_rate": 2.097338828083448e-05,
"loss": 0.0013,
"step": 3100
},
{
"epoch": 3.2,
"grad_norm": 0.0292133130133152,
"learning_rate": 1.987010641440447e-05,
"loss": 0.1935,
"step": 3200
},
{
"epoch": 3.3,
"grad_norm": 0.9771420955657959,
"learning_rate": 1.876682454797446e-05,
"loss": 0.0026,
"step": 3300
},
{
"epoch": 3.4,
"grad_norm": 0.13967016339302063,
"learning_rate": 1.7663542681544454e-05,
"loss": 0.0042,
"step": 3400
},
{
"epoch": 3.5,
"grad_norm": 44.07572937011719,
"learning_rate": 1.6560260815114444e-05,
"loss": 0.0002,
"step": 3500
},
{
"epoch": 3.6,
"grad_norm": 4.053230285644531,
"learning_rate": 1.545697894868443e-05,
"loss": 0.0109,
"step": 3600
},
{
"epoch": 3.7,
"grad_norm": 2.6551458835601807,
"learning_rate": 1.4353697082254424e-05,
"loss": 0.0001,
"step": 3700
},
{
"epoch": 3.8,
"grad_norm": 0.14367616176605225,
"learning_rate": 1.3250415215824414e-05,
"loss": 0.0001,
"step": 3800
},
{
"epoch": 3.9,
"grad_norm": 0.05590776354074478,
"learning_rate": 1.2147133349394404e-05,
"loss": 0.0086,
"step": 3900
},
{
"epoch": 4.0,
"grad_norm": 0.6181541681289673,
"learning_rate": 1.1043851482964394e-05,
"loss": 0.0789,
"step": 4000
},
{
"epoch": 4.0,
"eval_economic_inequality_accuracy": 0.8123425692695214,
"eval_economic_inequality_f1": 0.8054350386595589,
"eval_economic_policy_benefits_accuracy": 0.8518518518518519,
"eval_economic_policy_benefits_f1": 0.8269803894803894,
"eval_ethnic_boundaries_accuracy": 0.9395424836601307,
"eval_ethnic_boundaries_f1": 0.9411192800317346,
"eval_language_policy_accuracy": 0.7058823529411765,
"eval_language_policy_f1": 0.6335511982570806,
"eval_loss": NaN,
"eval_mother_tongue_education_accuracy": 0.8888888888888888,
"eval_mother_tongue_education_f1": 0.837037037037037,
"eval_overall_accuracy": 0.8390163480974149,
"eval_overall_f1": 0.8135900008491362,
"eval_religion_ethnic_identity_accuracy": 0.8355899419729207,
"eval_religion_ethnic_identity_f1": 0.8374170616290167,
"eval_runtime": 5.6175,
"eval_samples_per_second": 712.064,
"eval_steps_per_second": 44.504,
"step": 4000
}
],
"logging_steps": 100,
"max_steps": 5000,
"num_input_tokens_seen": 0,
"num_train_epochs": 5,
"save_steps": 500,
"stateful_callbacks": {
"EarlyStoppingCallback": {
"args": {
"early_stopping_patience": 2,
"early_stopping_threshold": 0.0
},
"attributes": {
"early_stopping_patience_counter": 0
}
},
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1.6841526607872e+16,
"train_batch_size": 16,
"trial_name": null,
"trial_params": {
"gradient_accumulation_steps": 1,
"learning_rate": 4.452845612911518e-05,
"num_train_epochs": 5,
"per_device_train_batch_size": 16,
"warmup_steps": 964,
"weight_decay": 0.08427803271887896
}
}