YeBhoneLin10's picture
Training in progress, step 2000, checkpoint
9bcf96f verified
{
"best_metric": 53.553446508933746,
"best_model_checkpoint": "./checkpoint-2000",
"epoch": 5.01375,
"eval_steps": 1000,
"global_step": 2000,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.00625,
"grad_norm": 15.778736114501953,
"learning_rate": 4.6000000000000004e-07,
"loss": 1.5646,
"step": 25
},
{
"epoch": 0.0125,
"grad_norm": 5.086428165435791,
"learning_rate": 9.600000000000001e-07,
"loss": 1.3553,
"step": 50
},
{
"epoch": 0.01875,
"grad_norm": 4.45254373550415,
"learning_rate": 1.46e-06,
"loss": 1.1549,
"step": 75
},
{
"epoch": 0.025,
"grad_norm": 4.353440284729004,
"learning_rate": 1.9600000000000003e-06,
"loss": 1.0095,
"step": 100
},
{
"epoch": 0.03125,
"grad_norm": 4.452302932739258,
"learning_rate": 2.46e-06,
"loss": 0.9047,
"step": 125
},
{
"epoch": 0.0375,
"grad_norm": 6.658376693725586,
"learning_rate": 2.96e-06,
"loss": 0.8367,
"step": 150
},
{
"epoch": 0.04375,
"grad_norm": 6.855594158172607,
"learning_rate": 3.46e-06,
"loss": 0.7943,
"step": 175
},
{
"epoch": 0.05,
"grad_norm": 5.464341640472412,
"learning_rate": 3.96e-06,
"loss": 0.7727,
"step": 200
},
{
"epoch": 0.05625,
"grad_norm": 6.465638160705566,
"learning_rate": 4.440000000000001e-06,
"loss": 0.7472,
"step": 225
},
{
"epoch": 0.0625,
"grad_norm": 7.32954740524292,
"learning_rate": 4.94e-06,
"loss": 0.7267,
"step": 250
},
{
"epoch": 0.06875,
"grad_norm": 7.171662330627441,
"learning_rate": 5.4400000000000004e-06,
"loss": 0.7382,
"step": 275
},
{
"epoch": 0.075,
"grad_norm": 10.485993385314941,
"learning_rate": 5.94e-06,
"loss": 0.6721,
"step": 300
},
{
"epoch": 0.08125,
"grad_norm": 7.209652423858643,
"learning_rate": 6.440000000000001e-06,
"loss": 0.6367,
"step": 325
},
{
"epoch": 0.0875,
"grad_norm": 9.550680160522461,
"learning_rate": 6.9400000000000005e-06,
"loss": 0.5658,
"step": 350
},
{
"epoch": 0.09375,
"grad_norm": 8.483597755432129,
"learning_rate": 7.440000000000001e-06,
"loss": 0.4797,
"step": 375
},
{
"epoch": 1.00275,
"grad_norm": 7.711348056793213,
"learning_rate": 7.94e-06,
"loss": 0.4997,
"step": 400
},
{
"epoch": 1.009,
"grad_norm": 8.237557411193848,
"learning_rate": 8.44e-06,
"loss": 0.4414,
"step": 425
},
{
"epoch": 1.01525,
"grad_norm": 7.03880500793457,
"learning_rate": 8.94e-06,
"loss": 0.3393,
"step": 450
},
{
"epoch": 1.0215,
"grad_norm": 4.42921257019043,
"learning_rate": 9.440000000000001e-06,
"loss": 0.2744,
"step": 475
},
{
"epoch": 1.02775,
"grad_norm": 4.717575550079346,
"learning_rate": 9.940000000000001e-06,
"loss": 0.237,
"step": 500
},
{
"epoch": 1.034,
"grad_norm": 4.850898742675781,
"learning_rate": 9.937142857142858e-06,
"loss": 0.2114,
"step": 525
},
{
"epoch": 1.04025,
"grad_norm": 3.328772783279419,
"learning_rate": 9.865714285714285e-06,
"loss": 0.1906,
"step": 550
},
{
"epoch": 1.0465,
"grad_norm": 4.541759014129639,
"learning_rate": 9.794285714285714e-06,
"loss": 0.1785,
"step": 575
},
{
"epoch": 1.05275,
"grad_norm": 3.4614176750183105,
"learning_rate": 9.722857142857143e-06,
"loss": 0.1978,
"step": 600
},
{
"epoch": 1.059,
"grad_norm": 3.4890570640563965,
"learning_rate": 9.651428571428572e-06,
"loss": 0.1822,
"step": 625
},
{
"epoch": 1.06525,
"grad_norm": 4.555150032043457,
"learning_rate": 9.58e-06,
"loss": 0.1687,
"step": 650
},
{
"epoch": 1.0715,
"grad_norm": 3.379296064376831,
"learning_rate": 9.508571428571429e-06,
"loss": 0.1563,
"step": 675
},
{
"epoch": 1.07775,
"grad_norm": 3.8745622634887695,
"learning_rate": 9.437142857142858e-06,
"loss": 0.1229,
"step": 700
},
{
"epoch": 1.084,
"grad_norm": 3.2507641315460205,
"learning_rate": 9.365714285714287e-06,
"loss": 0.1202,
"step": 725
},
{
"epoch": 1.09025,
"grad_norm": 2.579324960708618,
"learning_rate": 9.294285714285714e-06,
"loss": 0.1309,
"step": 750
},
{
"epoch": 1.0965,
"grad_norm": 3.5643343925476074,
"learning_rate": 9.222857142857143e-06,
"loss": 0.1206,
"step": 775
},
{
"epoch": 2.0055,
"grad_norm": 2.8714065551757812,
"learning_rate": 9.151428571428572e-06,
"loss": 0.1458,
"step": 800
},
{
"epoch": 2.01175,
"grad_norm": 2.573326826095581,
"learning_rate": 9.080000000000001e-06,
"loss": 0.1239,
"step": 825
},
{
"epoch": 2.018,
"grad_norm": 1.7793731689453125,
"learning_rate": 9.00857142857143e-06,
"loss": 0.111,
"step": 850
},
{
"epoch": 2.02425,
"grad_norm": 1.763602614402771,
"learning_rate": 8.937142857142857e-06,
"loss": 0.1054,
"step": 875
},
{
"epoch": 2.0305,
"grad_norm": 1.9787425994873047,
"learning_rate": 8.865714285714287e-06,
"loss": 0.0982,
"step": 900
},
{
"epoch": 2.03675,
"grad_norm": 2.2583250999450684,
"learning_rate": 8.794285714285716e-06,
"loss": 0.0957,
"step": 925
},
{
"epoch": 2.043,
"grad_norm": 2.4571313858032227,
"learning_rate": 8.722857142857145e-06,
"loss": 0.0879,
"step": 950
},
{
"epoch": 2.04925,
"grad_norm": 2.555344581604004,
"learning_rate": 8.651428571428572e-06,
"loss": 0.0925,
"step": 975
},
{
"epoch": 2.0555,
"grad_norm": 2.3653111457824707,
"learning_rate": 8.580000000000001e-06,
"loss": 0.1087,
"step": 1000
},
{
"epoch": 2.0555,
"eval_loss": 0.13476958870887756,
"eval_runtime": 669.8451,
"eval_samples_per_second": 0.951,
"eval_steps_per_second": 0.119,
"eval_wer": 57.21605845920777,
"step": 1000
},
{
"epoch": 2.06175,
"grad_norm": 3.215674638748169,
"learning_rate": 8.50857142857143e-06,
"loss": 0.0907,
"step": 1025
},
{
"epoch": 2.068,
"grad_norm": 2.3883891105651855,
"learning_rate": 8.437142857142859e-06,
"loss": 0.0876,
"step": 1050
},
{
"epoch": 2.07425,
"grad_norm": 1.890366792678833,
"learning_rate": 8.365714285714286e-06,
"loss": 0.0724,
"step": 1075
},
{
"epoch": 2.0805,
"grad_norm": 2.6750569343566895,
"learning_rate": 8.294285714285715e-06,
"loss": 0.0702,
"step": 1100
},
{
"epoch": 2.08675,
"grad_norm": 2.3613970279693604,
"learning_rate": 8.222857142857144e-06,
"loss": 0.0768,
"step": 1125
},
{
"epoch": 2.093,
"grad_norm": 2.880221128463745,
"learning_rate": 8.151428571428572e-06,
"loss": 0.0653,
"step": 1150
},
{
"epoch": 3.002,
"grad_norm": 2.5774643421173096,
"learning_rate": 8.08e-06,
"loss": 0.0863,
"step": 1175
},
{
"epoch": 3.00825,
"grad_norm": 2.212843894958496,
"learning_rate": 8.00857142857143e-06,
"loss": 0.0786,
"step": 1200
},
{
"epoch": 3.0145,
"grad_norm": 1.7202649116516113,
"learning_rate": 7.937142857142857e-06,
"loss": 0.0695,
"step": 1225
},
{
"epoch": 3.02075,
"grad_norm": 1.4540152549743652,
"learning_rate": 7.865714285714286e-06,
"loss": 0.062,
"step": 1250
},
{
"epoch": 3.027,
"grad_norm": 1.4593051671981812,
"learning_rate": 7.794285714285715e-06,
"loss": 0.0586,
"step": 1275
},
{
"epoch": 3.03325,
"grad_norm": 1.5141732692718506,
"learning_rate": 7.722857142857142e-06,
"loss": 0.0555,
"step": 1300
},
{
"epoch": 3.0395,
"grad_norm": 1.751680850982666,
"learning_rate": 7.651428571428571e-06,
"loss": 0.0572,
"step": 1325
},
{
"epoch": 3.04575,
"grad_norm": 1.7433584928512573,
"learning_rate": 7.58e-06,
"loss": 0.0514,
"step": 1350
},
{
"epoch": 3.052,
"grad_norm": 1.5257874727249146,
"learning_rate": 7.508571428571429e-06,
"loss": 0.064,
"step": 1375
},
{
"epoch": 3.05825,
"grad_norm": 1.4389644861221313,
"learning_rate": 7.4371428571428575e-06,
"loss": 0.0633,
"step": 1400
},
{
"epoch": 3.0645,
"grad_norm": 2.6773693561553955,
"learning_rate": 7.365714285714286e-06,
"loss": 0.0529,
"step": 1425
},
{
"epoch": 3.07075,
"grad_norm": 2.163686752319336,
"learning_rate": 7.294285714285715e-06,
"loss": 0.0567,
"step": 1450
},
{
"epoch": 3.077,
"grad_norm": 2.352548599243164,
"learning_rate": 7.222857142857144e-06,
"loss": 0.0378,
"step": 1475
},
{
"epoch": 3.08325,
"grad_norm": 2.7124457359313965,
"learning_rate": 7.151428571428573e-06,
"loss": 0.0468,
"step": 1500
},
{
"epoch": 3.0895,
"grad_norm": 2.316270112991333,
"learning_rate": 7.08e-06,
"loss": 0.047,
"step": 1525
},
{
"epoch": 3.09575,
"grad_norm": 4.7504963874816895,
"learning_rate": 7.008571428571429e-06,
"loss": 0.041,
"step": 1550
},
{
"epoch": 4.00475,
"grad_norm": 1.6336088180541992,
"learning_rate": 6.937142857142858e-06,
"loss": 0.061,
"step": 1575
},
{
"epoch": 4.011,
"grad_norm": 1.1898696422576904,
"learning_rate": 6.865714285714287e-06,
"loss": 0.0457,
"step": 1600
},
{
"epoch": 4.01725,
"grad_norm": 1.5887372493743896,
"learning_rate": 6.794285714285714e-06,
"loss": 0.0411,
"step": 1625
},
{
"epoch": 4.0235,
"grad_norm": 1.7909950017929077,
"learning_rate": 6.722857142857143e-06,
"loss": 0.0373,
"step": 1650
},
{
"epoch": 4.02975,
"grad_norm": 1.1431634426116943,
"learning_rate": 6.651428571428572e-06,
"loss": 0.0343,
"step": 1675
},
{
"epoch": 4.036,
"grad_norm": 1.5267891883850098,
"learning_rate": 6.5800000000000005e-06,
"loss": 0.035,
"step": 1700
},
{
"epoch": 4.04225,
"grad_norm": 1.8040025234222412,
"learning_rate": 6.5085714285714295e-06,
"loss": 0.0312,
"step": 1725
},
{
"epoch": 4.0485,
"grad_norm": 1.672837257385254,
"learning_rate": 6.437142857142858e-06,
"loss": 0.034,
"step": 1750
},
{
"epoch": 4.05475,
"grad_norm": 1.5159598588943481,
"learning_rate": 6.365714285714286e-06,
"loss": 0.0431,
"step": 1775
},
{
"epoch": 4.061,
"grad_norm": 2.1531107425689697,
"learning_rate": 6.294285714285715e-06,
"loss": 0.0357,
"step": 1800
},
{
"epoch": 4.06725,
"grad_norm": 3.2716612815856934,
"learning_rate": 6.222857142857144e-06,
"loss": 0.0341,
"step": 1825
},
{
"epoch": 4.0735,
"grad_norm": 1.041003942489624,
"learning_rate": 6.151428571428571e-06,
"loss": 0.0302,
"step": 1850
},
{
"epoch": 4.07975,
"grad_norm": 2.5585858821868896,
"learning_rate": 6.08e-06,
"loss": 0.0269,
"step": 1875
},
{
"epoch": 4.086,
"grad_norm": 2.4382855892181396,
"learning_rate": 6.008571428571429e-06,
"loss": 0.0291,
"step": 1900
},
{
"epoch": 4.09225,
"grad_norm": 3.0103349685668945,
"learning_rate": 5.937142857142858e-06,
"loss": 0.0228,
"step": 1925
},
{
"epoch": 5.00125,
"grad_norm": 1.7492283582687378,
"learning_rate": 5.865714285714286e-06,
"loss": 0.0331,
"step": 1950
},
{
"epoch": 5.0075,
"grad_norm": 1.3919484615325928,
"learning_rate": 5.794285714285715e-06,
"loss": 0.0384,
"step": 1975
},
{
"epoch": 5.01375,
"grad_norm": 1.131212592124939,
"learning_rate": 5.722857142857144e-06,
"loss": 0.0277,
"step": 2000
},
{
"epoch": 5.01375,
"eval_loss": 0.12749460339546204,
"eval_runtime": 670.7043,
"eval_samples_per_second": 0.95,
"eval_steps_per_second": 0.119,
"eval_wer": 53.553446508933746,
"step": 2000
}
],
"logging_steps": 25,
"max_steps": 4000,
"num_input_tokens_seen": 0,
"num_train_epochs": 9223372036854775807,
"save_steps": 1000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 9.2217464672256e+18,
"train_batch_size": 16,
"trial_name": null,
"trial_params": null
}