| { |
| "best_global_step": 416, |
| "best_metric": 1.697021484375, |
| "best_model_checkpoint": "./tinyllama-shakespeare-lora/checkpoint-416", |
| "epoch": 5.0, |
| "eval_steps": 500, |
| "global_step": 520, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 2.237028416991234, |
| "epoch": 0.04819277108433735, |
| "grad_norm": 2.990326166152954, |
| "learning_rate": 8e-05, |
| "loss": 3.3300735473632814, |
| "mean_token_accuracy": 0.4388114660978317, |
| "num_tokens": 3544.0, |
| "step": 5 |
| }, |
| { |
| "entropy": 2.1227019637823106, |
| "epoch": 0.0963855421686747, |
| "grad_norm": 3.1520185470581055, |
| "learning_rate": 0.00018, |
| "loss": 3.040688896179199, |
| "mean_token_accuracy": 0.48041139617562295, |
| "num_tokens": 6921.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 2.470001167058945, |
| "epoch": 0.14457831325301204, |
| "grad_norm": 2.7241196632385254, |
| "learning_rate": 0.00019843137254901963, |
| "loss": 2.6650358200073243, |
| "mean_token_accuracy": 0.5012731336057186, |
| "num_tokens": 9897.0, |
| "step": 15 |
| }, |
| { |
| "entropy": 2.4187598139047624, |
| "epoch": 0.1927710843373494, |
| "grad_norm": 1.1179628372192383, |
| "learning_rate": 0.00019647058823529413, |
| "loss": 2.336894989013672, |
| "mean_token_accuracy": 0.544729282706976, |
| "num_tokens": 13386.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 2.4283315002918244, |
| "epoch": 0.24096385542168675, |
| "grad_norm": 1.0476402044296265, |
| "learning_rate": 0.00019450980392156863, |
| "loss": 2.257316017150879, |
| "mean_token_accuracy": 0.5755889780819416, |
| "num_tokens": 16770.0, |
| "step": 25 |
| }, |
| { |
| "entropy": 2.09900840818882, |
| "epoch": 0.2891566265060241, |
| "grad_norm": 1.005602240562439, |
| "learning_rate": 0.00019254901960784316, |
| "loss": 2.004672813415527, |
| "mean_token_accuracy": 0.6035298384726048, |
| "num_tokens": 19815.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 2.067019133269787, |
| "epoch": 0.3373493975903614, |
| "grad_norm": 1.3398667573928833, |
| "learning_rate": 0.00019058823529411766, |
| "loss": 2.0276206970214843, |
| "mean_token_accuracy": 0.5973991125822067, |
| "num_tokens": 22956.0, |
| "step": 35 |
| }, |
| { |
| "entropy": 2.0127717584371565, |
| "epoch": 0.3855421686746988, |
| "grad_norm": 1.3001172542572021, |
| "learning_rate": 0.00018862745098039216, |
| "loss": 1.989430046081543, |
| "mean_token_accuracy": 0.5942102774977684, |
| "num_tokens": 26147.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 1.948568905889988, |
| "epoch": 0.43373493975903615, |
| "grad_norm": 1.463757038116455, |
| "learning_rate": 0.0001866666666666667, |
| "loss": 2.002850341796875, |
| "mean_token_accuracy": 0.6077656075358391, |
| "num_tokens": 29290.0, |
| "step": 45 |
| }, |
| { |
| "entropy": 1.9544441923499107, |
| "epoch": 0.4819277108433735, |
| "grad_norm": 1.0674948692321777, |
| "learning_rate": 0.0001847058823529412, |
| "loss": 2.0039562225341796, |
| "mean_token_accuracy": 0.5964730970561505, |
| "num_tokens": 32774.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.991774895787239, |
| "epoch": 0.5301204819277109, |
| "grad_norm": 1.1058229207992554, |
| "learning_rate": 0.0001827450980392157, |
| "loss": 1.9143487930297851, |
| "mean_token_accuracy": 0.5965772472321987, |
| "num_tokens": 36361.0, |
| "step": 55 |
| }, |
| { |
| "entropy": 1.9300899803638458, |
| "epoch": 0.5783132530120482, |
| "grad_norm": 1.3332328796386719, |
| "learning_rate": 0.00018078431372549022, |
| "loss": 1.989386749267578, |
| "mean_token_accuracy": 0.595651201158762, |
| "num_tokens": 39883.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 1.8389213621616363, |
| "epoch": 0.6265060240963856, |
| "grad_norm": 1.600728988647461, |
| "learning_rate": 0.00017882352941176472, |
| "loss": 1.8484268188476562, |
| "mean_token_accuracy": 0.6202847644686699, |
| "num_tokens": 43257.0, |
| "step": 65 |
| }, |
| { |
| "entropy": 1.8422008126974105, |
| "epoch": 0.6746987951807228, |
| "grad_norm": 1.148055911064148, |
| "learning_rate": 0.00017686274509803922, |
| "loss": 1.846010971069336, |
| "mean_token_accuracy": 0.6175518661737442, |
| "num_tokens": 46204.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 1.8617600902915001, |
| "epoch": 0.7228915662650602, |
| "grad_norm": 1.3051718473434448, |
| "learning_rate": 0.00017490196078431375, |
| "loss": 1.877816390991211, |
| "mean_token_accuracy": 0.6046500727534294, |
| "num_tokens": 49793.0, |
| "step": 75 |
| }, |
| { |
| "entropy": 1.788547897338867, |
| "epoch": 0.7710843373493976, |
| "grad_norm": 1.6796694993972778, |
| "learning_rate": 0.00017294117647058825, |
| "loss": 1.8042177200317382, |
| "mean_token_accuracy": 0.626473106443882, |
| "num_tokens": 53104.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 1.7356875404715537, |
| "epoch": 0.8192771084337349, |
| "grad_norm": 1.6048922538757324, |
| "learning_rate": 0.00017098039215686275, |
| "loss": 1.7470354080200194, |
| "mean_token_accuracy": 0.6395883277058602, |
| "num_tokens": 56062.0, |
| "step": 85 |
| }, |
| { |
| "entropy": 1.7859284698963165, |
| "epoch": 0.8674698795180723, |
| "grad_norm": 1.2566527128219604, |
| "learning_rate": 0.00016901960784313725, |
| "loss": 1.8112169265747071, |
| "mean_token_accuracy": 0.6365099921822548, |
| "num_tokens": 59161.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 1.8159554556012154, |
| "epoch": 0.9156626506024096, |
| "grad_norm": 1.3626623153686523, |
| "learning_rate": 0.00016705882352941178, |
| "loss": 1.831540298461914, |
| "mean_token_accuracy": 0.6189694285392762, |
| "num_tokens": 62426.0, |
| "step": 95 |
| }, |
| { |
| "entropy": 1.7219859138131142, |
| "epoch": 0.963855421686747, |
| "grad_norm": 1.4669969081878662, |
| "learning_rate": 0.00016509803921568628, |
| "loss": 1.708040428161621, |
| "mean_token_accuracy": 0.6417586743831635, |
| "num_tokens": 65573.0, |
| "step": 100 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_entropy": 1.8488803276648889, |
| "eval_loss": 1.8482003211975098, |
| "eval_mean_token_accuracy": 0.6005064799235418, |
| "eval_num_tokens": 68001.0, |
| "eval_runtime": 31.0894, |
| "eval_samples_per_second": 6.69, |
| "eval_steps_per_second": 0.836, |
| "step": 104 |
| }, |
| { |
| "entropy": 1.703609773987218, |
| "epoch": 1.0096385542168675, |
| "grad_norm": 1.5622724294662476, |
| "learning_rate": 0.00016313725490196078, |
| "loss": 1.749994659423828, |
| "mean_token_accuracy": 0.6331688251934553, |
| "num_tokens": 68613.0, |
| "step": 105 |
| }, |
| { |
| "entropy": 1.841702875494957, |
| "epoch": 1.0578313253012048, |
| "grad_norm": 1.2482104301452637, |
| "learning_rate": 0.0001611764705882353, |
| "loss": 1.826130485534668, |
| "mean_token_accuracy": 0.618197824060917, |
| "num_tokens": 72379.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 1.7080746620893479, |
| "epoch": 1.106024096385542, |
| "grad_norm": 1.5051069259643555, |
| "learning_rate": 0.0001592156862745098, |
| "loss": 1.735462760925293, |
| "mean_token_accuracy": 0.6424251943826675, |
| "num_tokens": 75330.0, |
| "step": 115 |
| }, |
| { |
| "entropy": 1.7118461817502975, |
| "epoch": 1.1542168674698796, |
| "grad_norm": 1.4837011098861694, |
| "learning_rate": 0.0001572549019607843, |
| "loss": 1.7383131027221679, |
| "mean_token_accuracy": 0.637732881307602, |
| "num_tokens": 78552.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 1.7180065914988518, |
| "epoch": 1.202409638554217, |
| "grad_norm": 1.5369439125061035, |
| "learning_rate": 0.00015529411764705884, |
| "loss": 1.7133190155029296, |
| "mean_token_accuracy": 0.6356611765921116, |
| "num_tokens": 81798.0, |
| "step": 125 |
| }, |
| { |
| "entropy": 1.6460102945566177, |
| "epoch": 1.2506024096385542, |
| "grad_norm": 1.483790636062622, |
| "learning_rate": 0.00015333333333333334, |
| "loss": 1.598601722717285, |
| "mean_token_accuracy": 0.6597890131175518, |
| "num_tokens": 85117.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 1.7030983410775662, |
| "epoch": 1.2987951807228915, |
| "grad_norm": 1.3817803859710693, |
| "learning_rate": 0.00015137254901960784, |
| "loss": 1.7572540283203124, |
| "mean_token_accuracy": 0.6322377510368824, |
| "num_tokens": 88742.0, |
| "step": 135 |
| }, |
| { |
| "entropy": 1.7065479427576065, |
| "epoch": 1.346987951807229, |
| "grad_norm": 1.5301700830459595, |
| "learning_rate": 0.00014941176470588237, |
| "loss": 1.7399499893188477, |
| "mean_token_accuracy": 0.6498943358659744, |
| "num_tokens": 92210.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 1.625732569396496, |
| "epoch": 1.3951807228915662, |
| "grad_norm": 1.7736130952835083, |
| "learning_rate": 0.00014745098039215687, |
| "loss": 1.5962481498718262, |
| "mean_token_accuracy": 0.6658786356449127, |
| "num_tokens": 95303.0, |
| "step": 145 |
| }, |
| { |
| "entropy": 1.6358379781246186, |
| "epoch": 1.4433734939759035, |
| "grad_norm": 1.7948054075241089, |
| "learning_rate": 0.00014549019607843137, |
| "loss": 1.537781047821045, |
| "mean_token_accuracy": 0.6697517029941082, |
| "num_tokens": 98384.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 1.5426702469587326, |
| "epoch": 1.491566265060241, |
| "grad_norm": 1.9344528913497925, |
| "learning_rate": 0.0001435294117647059, |
| "loss": 1.623773193359375, |
| "mean_token_accuracy": 0.6632205478847026, |
| "num_tokens": 101490.0, |
| "step": 155 |
| }, |
| { |
| "entropy": 1.52632287889719, |
| "epoch": 1.5397590361445783, |
| "grad_norm": 1.7781139612197876, |
| "learning_rate": 0.0001415686274509804, |
| "loss": 1.5219478607177734, |
| "mean_token_accuracy": 0.6700925581157208, |
| "num_tokens": 104191.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 1.5876709207892419, |
| "epoch": 1.5879518072289156, |
| "grad_norm": 1.6688432693481445, |
| "learning_rate": 0.0001396078431372549, |
| "loss": 1.6492792129516602, |
| "mean_token_accuracy": 0.6539962455630303, |
| "num_tokens": 107261.0, |
| "step": 165 |
| }, |
| { |
| "entropy": 1.670412401854992, |
| "epoch": 1.636144578313253, |
| "grad_norm": 1.8056694269180298, |
| "learning_rate": 0.00013764705882352943, |
| "loss": 1.7233877182006836, |
| "mean_token_accuracy": 0.6458830818533897, |
| "num_tokens": 110608.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 1.6271757155656814, |
| "epoch": 1.6843373493975904, |
| "grad_norm": 1.6283652782440186, |
| "learning_rate": 0.00013568627450980393, |
| "loss": 1.6276430130004882, |
| "mean_token_accuracy": 0.6651988372206687, |
| "num_tokens": 113881.0, |
| "step": 175 |
| }, |
| { |
| "entropy": 1.6719022899866105, |
| "epoch": 1.7325301204819277, |
| "grad_norm": 1.7494244575500488, |
| "learning_rate": 0.00013372549019607843, |
| "loss": 1.6668785095214844, |
| "mean_token_accuracy": 0.6384630501270294, |
| "num_tokens": 117044.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 1.7114125609397888, |
| "epoch": 1.7807228915662652, |
| "grad_norm": 1.5748646259307861, |
| "learning_rate": 0.00013176470588235296, |
| "loss": 1.6835250854492188, |
| "mean_token_accuracy": 0.6382554531097412, |
| "num_tokens": 120606.0, |
| "step": 185 |
| }, |
| { |
| "entropy": 1.575966164469719, |
| "epoch": 1.8289156626506025, |
| "grad_norm": 1.5379868745803833, |
| "learning_rate": 0.00012980392156862746, |
| "loss": 1.6510162353515625, |
| "mean_token_accuracy": 0.6542762629687786, |
| "num_tokens": 123899.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 1.6397091552615166, |
| "epoch": 1.8771084337349397, |
| "grad_norm": 1.8050676584243774, |
| "learning_rate": 0.00012784313725490196, |
| "loss": 1.6907238006591796, |
| "mean_token_accuracy": 0.6459958925843239, |
| "num_tokens": 127473.0, |
| "step": 195 |
| }, |
| { |
| "entropy": 1.6371865630149842, |
| "epoch": 1.9253012048192772, |
| "grad_norm": 1.7931848764419556, |
| "learning_rate": 0.0001258823529411765, |
| "loss": 1.680709457397461, |
| "mean_token_accuracy": 0.6382987394928932, |
| "num_tokens": 130771.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 1.6472558170557021, |
| "epoch": 1.9734939759036143, |
| "grad_norm": 1.731370210647583, |
| "learning_rate": 0.000123921568627451, |
| "loss": 1.6797250747680663, |
| "mean_token_accuracy": 0.6526046924293041, |
| "num_tokens": 134125.0, |
| "step": 205 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_entropy": 1.8006169016544635, |
| "eval_loss": 1.7452114820480347, |
| "eval_mean_token_accuracy": 0.6168445830161755, |
| "eval_num_tokens": 136002.0, |
| "eval_runtime": 31.081, |
| "eval_samples_per_second": 6.692, |
| "eval_steps_per_second": 0.837, |
| "step": 208 |
| }, |
| { |
| "entropy": 1.6643306487484981, |
| "epoch": 2.019277108433735, |
| "grad_norm": 1.6965025663375854, |
| "learning_rate": 0.0001219607843137255, |
| "loss": 1.658614730834961, |
| "mean_token_accuracy": 0.6498823793310868, |
| "num_tokens": 137377.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 1.6733600363135337, |
| "epoch": 2.067469879518072, |
| "grad_norm": 1.8712608814239502, |
| "learning_rate": 0.00012, |
| "loss": 1.5869686126708984, |
| "mean_token_accuracy": 0.656411537528038, |
| "num_tokens": 141109.0, |
| "step": 215 |
| }, |
| { |
| "entropy": 1.5299664348363877, |
| "epoch": 2.1156626506024097, |
| "grad_norm": 2.0042712688446045, |
| "learning_rate": 0.00011803921568627452, |
| "loss": 1.501875877380371, |
| "mean_token_accuracy": 0.6718885555863381, |
| "num_tokens": 144100.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 1.5708236396312714, |
| "epoch": 2.163855421686747, |
| "grad_norm": 2.0190634727478027, |
| "learning_rate": 0.00011607843137254903, |
| "loss": 1.6497461318969726, |
| "mean_token_accuracy": 0.6537894055247306, |
| "num_tokens": 147584.0, |
| "step": 225 |
| }, |
| { |
| "entropy": 1.5232929274439813, |
| "epoch": 2.212048192771084, |
| "grad_norm": 1.9235641956329346, |
| "learning_rate": 0.00011411764705882353, |
| "loss": 1.6025299072265624, |
| "mean_token_accuracy": 0.6778546750545502, |
| "num_tokens": 150511.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 1.6279756397008895, |
| "epoch": 2.2602409638554217, |
| "grad_norm": 1.8622772693634033, |
| "learning_rate": 0.00011215686274509805, |
| "loss": 1.546085739135742, |
| "mean_token_accuracy": 0.6725640386343003, |
| "num_tokens": 154053.0, |
| "step": 235 |
| }, |
| { |
| "entropy": 1.5271618023514748, |
| "epoch": 2.3084337349397592, |
| "grad_norm": 2.066518545150757, |
| "learning_rate": 0.00011019607843137256, |
| "loss": 1.5498031616210937, |
| "mean_token_accuracy": 0.6706199154257775, |
| "num_tokens": 157138.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 1.5382411390542985, |
| "epoch": 2.3566265060240963, |
| "grad_norm": 1.9565099477767944, |
| "learning_rate": 0.00010823529411764706, |
| "loss": 1.4900429725646973, |
| "mean_token_accuracy": 0.6834538638591766, |
| "num_tokens": 160436.0, |
| "step": 245 |
| }, |
| { |
| "entropy": 1.5565499082207679, |
| "epoch": 2.404819277108434, |
| "grad_norm": 1.769296646118164, |
| "learning_rate": 0.00010627450980392158, |
| "loss": 1.5247212409973145, |
| "mean_token_accuracy": 0.6715260729193687, |
| "num_tokens": 163761.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 1.560780017077923, |
| "epoch": 2.453012048192771, |
| "grad_norm": 1.9404278993606567, |
| "learning_rate": 0.00010431372549019609, |
| "loss": 1.4871397018432617, |
| "mean_token_accuracy": 0.6675436168909072, |
| "num_tokens": 166898.0, |
| "step": 255 |
| }, |
| { |
| "entropy": 1.4660954281687737, |
| "epoch": 2.5012048192771084, |
| "grad_norm": 2.1916701793670654, |
| "learning_rate": 0.00010235294117647058, |
| "loss": 1.5004875183105468, |
| "mean_token_accuracy": 0.6763247601687908, |
| "num_tokens": 169909.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 1.577199426293373, |
| "epoch": 2.549397590361446, |
| "grad_norm": 1.8109275102615356, |
| "learning_rate": 0.00010039215686274511, |
| "loss": 1.5327059745788574, |
| "mean_token_accuracy": 0.6665430299937725, |
| "num_tokens": 173365.0, |
| "step": 265 |
| }, |
| { |
| "entropy": 1.5943815588951111, |
| "epoch": 2.597590361445783, |
| "grad_norm": 2.2097110748291016, |
| "learning_rate": 9.843137254901961e-05, |
| "loss": 1.629846954345703, |
| "mean_token_accuracy": 0.663162975013256, |
| "num_tokens": 176834.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 1.55985931456089, |
| "epoch": 2.6457831325301204, |
| "grad_norm": 1.896395206451416, |
| "learning_rate": 9.647058823529412e-05, |
| "loss": 1.4793538093566894, |
| "mean_token_accuracy": 0.6789445534348488, |
| "num_tokens": 180153.0, |
| "step": 275 |
| }, |
| { |
| "entropy": 1.6425225555896759, |
| "epoch": 2.693975903614458, |
| "grad_norm": 1.6390873193740845, |
| "learning_rate": 9.450980392156862e-05, |
| "loss": 1.5848325729370116, |
| "mean_token_accuracy": 0.6619467623531818, |
| "num_tokens": 183512.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 1.45964814722538, |
| "epoch": 2.742168674698795, |
| "grad_norm": 1.9988118410110474, |
| "learning_rate": 9.254901960784315e-05, |
| "loss": 1.46427001953125, |
| "mean_token_accuracy": 0.6810511723160744, |
| "num_tokens": 186472.0, |
| "step": 285 |
| }, |
| { |
| "entropy": 1.4157467983663081, |
| "epoch": 2.7903614457831325, |
| "grad_norm": 2.046511173248291, |
| "learning_rate": 9.058823529411765e-05, |
| "loss": 1.42630615234375, |
| "mean_token_accuracy": 0.7006928496062755, |
| "num_tokens": 189517.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 1.476995500922203, |
| "epoch": 2.83855421686747, |
| "grad_norm": 2.3397531509399414, |
| "learning_rate": 8.862745098039215e-05, |
| "loss": 1.495310115814209, |
| "mean_token_accuracy": 0.6897226825356484, |
| "num_tokens": 192652.0, |
| "step": 295 |
| }, |
| { |
| "entropy": 1.4484579890966416, |
| "epoch": 2.886746987951807, |
| "grad_norm": 2.0256145000457764, |
| "learning_rate": 8.666666666666667e-05, |
| "loss": 1.5049756050109864, |
| "mean_token_accuracy": 0.6935365840792656, |
| "num_tokens": 195757.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 1.4375765472650528, |
| "epoch": 2.9349397590361446, |
| "grad_norm": 1.795584797859192, |
| "learning_rate": 8.470588235294118e-05, |
| "loss": 1.4198492050170899, |
| "mean_token_accuracy": 0.6847325280308724, |
| "num_tokens": 198807.0, |
| "step": 305 |
| }, |
| { |
| "entropy": 1.528662195801735, |
| "epoch": 2.983132530120482, |
| "grad_norm": 2.0084290504455566, |
| "learning_rate": 8.274509803921568e-05, |
| "loss": 1.565225315093994, |
| "mean_token_accuracy": 0.6660942971706391, |
| "num_tokens": 202500.0, |
| "step": 310 |
| }, |
| { |
| "epoch": 3.0, |
| "eval_entropy": 1.6572299553797796, |
| "eval_loss": 1.710593342781067, |
| "eval_mean_token_accuracy": 0.6203672221073737, |
| "eval_num_tokens": 204003.0, |
| "eval_runtime": 31.0505, |
| "eval_samples_per_second": 6.699, |
| "eval_steps_per_second": 0.837, |
| "step": 312 |
| }, |
| { |
| "entropy": 1.5311788932273263, |
| "epoch": 3.0289156626506024, |
| "grad_norm": 1.8857117891311646, |
| "learning_rate": 8.07843137254902e-05, |
| "loss": 1.5346615791320801, |
| "mean_token_accuracy": 0.6760246863490657, |
| "num_tokens": 205682.0, |
| "step": 315 |
| }, |
| { |
| "entropy": 1.5602496802806853, |
| "epoch": 3.07710843373494, |
| "grad_norm": 2.1312615871429443, |
| "learning_rate": 7.882352941176471e-05, |
| "loss": 1.4694817543029786, |
| "mean_token_accuracy": 0.6782874375581741, |
| "num_tokens": 209067.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 1.5345655620098113, |
| "epoch": 3.125301204819277, |
| "grad_norm": 1.8577815294265747, |
| "learning_rate": 7.686274509803923e-05, |
| "loss": 1.4558188438415527, |
| "mean_token_accuracy": 0.6776880241930485, |
| "num_tokens": 212352.0, |
| "step": 325 |
| }, |
| { |
| "entropy": 1.4676252231001854, |
| "epoch": 3.1734939759036145, |
| "grad_norm": 2.027308940887451, |
| "learning_rate": 7.490196078431373e-05, |
| "loss": 1.4121692657470704, |
| "mean_token_accuracy": 0.6886831521987915, |
| "num_tokens": 215682.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 1.4628461375832558, |
| "epoch": 3.221686746987952, |
| "grad_norm": 1.9528241157531738, |
| "learning_rate": 7.294117647058823e-05, |
| "loss": 1.439143466949463, |
| "mean_token_accuracy": 0.6888918332755566, |
| "num_tokens": 219099.0, |
| "step": 335 |
| }, |
| { |
| "entropy": 1.3313937097787858, |
| "epoch": 3.269879518072289, |
| "grad_norm": 1.9298614263534546, |
| "learning_rate": 7.098039215686276e-05, |
| "loss": 1.380887508392334, |
| "mean_token_accuracy": 0.7101273387670517, |
| "num_tokens": 222041.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 1.5036465510725976, |
| "epoch": 3.3180722891566266, |
| "grad_norm": 2.5983974933624268, |
| "learning_rate": 6.901960784313726e-05, |
| "loss": 1.479258918762207, |
| "mean_token_accuracy": 0.6855721130967141, |
| "num_tokens": 225295.0, |
| "step": 345 |
| }, |
| { |
| "entropy": 1.4219771109521389, |
| "epoch": 3.3662650602409636, |
| "grad_norm": 2.4156622886657715, |
| "learning_rate": 6.705882352941176e-05, |
| "loss": 1.4329577445983888, |
| "mean_token_accuracy": 0.6899721816182136, |
| "num_tokens": 228520.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 1.526054358482361, |
| "epoch": 3.414457831325301, |
| "grad_norm": 2.509209156036377, |
| "learning_rate": 6.509803921568627e-05, |
| "loss": 1.4574121475219726, |
| "mean_token_accuracy": 0.6843675896525383, |
| "num_tokens": 231944.0, |
| "step": 355 |
| }, |
| { |
| "entropy": 1.462461456656456, |
| "epoch": 3.4626506024096386, |
| "grad_norm": 1.98367178440094, |
| "learning_rate": 6.313725490196079e-05, |
| "loss": 1.4001091957092284, |
| "mean_token_accuracy": 0.6919049903750419, |
| "num_tokens": 235338.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 1.4600355744361877, |
| "epoch": 3.5108433734939757, |
| "grad_norm": 2.3915727138519287, |
| "learning_rate": 6.11764705882353e-05, |
| "loss": 1.504780101776123, |
| "mean_token_accuracy": 0.6843942984938621, |
| "num_tokens": 238515.0, |
| "step": 365 |
| }, |
| { |
| "entropy": 1.3431835047900678, |
| "epoch": 3.559036144578313, |
| "grad_norm": 2.4486849308013916, |
| "learning_rate": 5.921568627450981e-05, |
| "loss": 1.3256868362426757, |
| "mean_token_accuracy": 0.7202241063117981, |
| "num_tokens": 241407.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 1.5020532310009003, |
| "epoch": 3.6072289156626507, |
| "grad_norm": 2.259896993637085, |
| "learning_rate": 5.725490196078431e-05, |
| "loss": 1.4902771949768066, |
| "mean_token_accuracy": 0.6747317329049111, |
| "num_tokens": 244810.0, |
| "step": 375 |
| }, |
| { |
| "entropy": 1.449581353366375, |
| "epoch": 3.6554216867469878, |
| "grad_norm": 2.306088924407959, |
| "learning_rate": 5.529411764705883e-05, |
| "loss": 1.4183819770812989, |
| "mean_token_accuracy": 0.6943565353751182, |
| "num_tokens": 247917.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 1.401406754553318, |
| "epoch": 3.7036144578313253, |
| "grad_norm": 2.6169755458831787, |
| "learning_rate": 5.333333333333333e-05, |
| "loss": 1.324946975708008, |
| "mean_token_accuracy": 0.7123707890510559, |
| "num_tokens": 251035.0, |
| "step": 385 |
| }, |
| { |
| "entropy": 1.3621296539902688, |
| "epoch": 3.7518072289156628, |
| "grad_norm": 2.3834919929504395, |
| "learning_rate": 5.137254901960784e-05, |
| "loss": 1.4367511749267579, |
| "mean_token_accuracy": 0.6939964599907398, |
| "num_tokens": 254564.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 1.4182414807379247, |
| "epoch": 3.8, |
| "grad_norm": 2.1327056884765625, |
| "learning_rate": 4.9411764705882355e-05, |
| "loss": 1.396888542175293, |
| "mean_token_accuracy": 0.6919433787465096, |
| "num_tokens": 258001.0, |
| "step": 395 |
| }, |
| { |
| "entropy": 1.4946724489331245, |
| "epoch": 3.8481927710843373, |
| "grad_norm": 2.383056640625, |
| "learning_rate": 4.745098039215686e-05, |
| "loss": 1.3979401588439941, |
| "mean_token_accuracy": 0.695962718129158, |
| "num_tokens": 260974.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 1.449859108030796, |
| "epoch": 3.896385542168675, |
| "grad_norm": 2.490652084350586, |
| "learning_rate": 4.549019607843137e-05, |
| "loss": 1.4903193473815919, |
| "mean_token_accuracy": 0.686176010966301, |
| "num_tokens": 264415.0, |
| "step": 405 |
| }, |
| { |
| "entropy": 1.4792103588581085, |
| "epoch": 3.944578313253012, |
| "grad_norm": 2.542536973953247, |
| "learning_rate": 4.3529411764705885e-05, |
| "loss": 1.4710602760314941, |
| "mean_token_accuracy": 0.6943801581859589, |
| "num_tokens": 267945.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 1.4717445507645608, |
| "epoch": 3.9927710843373494, |
| "grad_norm": 2.2224795818328857, |
| "learning_rate": 4.156862745098039e-05, |
| "loss": 1.4788040161132812, |
| "mean_token_accuracy": 0.6895710974931717, |
| "num_tokens": 271493.0, |
| "step": 415 |
| }, |
| { |
| "epoch": 4.0, |
| "eval_entropy": 1.620164128450247, |
| "eval_loss": 1.697021484375, |
| "eval_mean_token_accuracy": 0.6267852737353399, |
| "eval_num_tokens": 272004.0, |
| "eval_runtime": 31.0743, |
| "eval_samples_per_second": 6.694, |
| "eval_steps_per_second": 0.837, |
| "step": 416 |
| }, |
| { |
| "entropy": 1.4130519565783048, |
| "epoch": 4.03855421686747, |
| "grad_norm": 2.297147274017334, |
| "learning_rate": 3.96078431372549e-05, |
| "loss": 1.429203701019287, |
| "mean_token_accuracy": 0.6895033309334203, |
| "num_tokens": 274843.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 1.3913781866431236, |
| "epoch": 4.086746987951807, |
| "grad_norm": 2.412026882171631, |
| "learning_rate": 3.7647058823529415e-05, |
| "loss": 1.2258234977722169, |
| "mean_token_accuracy": 0.7282624199986458, |
| "num_tokens": 277622.0, |
| "step": 425 |
| }, |
| { |
| "entropy": 1.4595662340521813, |
| "epoch": 4.134939759036144, |
| "grad_norm": 2.2476892471313477, |
| "learning_rate": 3.568627450980392e-05, |
| "loss": 1.3860394477844238, |
| "mean_token_accuracy": 0.6948064118623734, |
| "num_tokens": 281115.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 1.4310455292463302, |
| "epoch": 4.183132530120482, |
| "grad_norm": 1.9430752992630005, |
| "learning_rate": 3.372549019607844e-05, |
| "loss": 1.4230783462524415, |
| "mean_token_accuracy": 0.7034219011664391, |
| "num_tokens": 284784.0, |
| "step": 435 |
| }, |
| { |
| "entropy": 1.3278866931796074, |
| "epoch": 4.231325301204819, |
| "grad_norm": 1.9049440622329712, |
| "learning_rate": 3.176470588235294e-05, |
| "loss": 1.424240779876709, |
| "mean_token_accuracy": 0.7034214176237583, |
| "num_tokens": 288256.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 1.3592324391007424, |
| "epoch": 4.279518072289156, |
| "grad_norm": 2.3773999214172363, |
| "learning_rate": 2.9803921568627453e-05, |
| "loss": 1.3538729667663574, |
| "mean_token_accuracy": 0.7138303920626641, |
| "num_tokens": 291712.0, |
| "step": 445 |
| }, |
| { |
| "entropy": 1.4115683421492577, |
| "epoch": 4.327710843373494, |
| "grad_norm": 2.435506820678711, |
| "learning_rate": 2.7843137254901964e-05, |
| "loss": 1.3001550674438476, |
| "mean_token_accuracy": 0.7063792496919632, |
| "num_tokens": 294817.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 1.4070537582039833, |
| "epoch": 4.375903614457831, |
| "grad_norm": 2.1623501777648926, |
| "learning_rate": 2.5882352941176475e-05, |
| "loss": 1.396212100982666, |
| "mean_token_accuracy": 0.6988388016819954, |
| "num_tokens": 298135.0, |
| "step": 455 |
| }, |
| { |
| "entropy": 1.3870512664318084, |
| "epoch": 4.424096385542168, |
| "grad_norm": 2.5376806259155273, |
| "learning_rate": 2.3921568627450983e-05, |
| "loss": 1.3213976860046386, |
| "mean_token_accuracy": 0.712082713842392, |
| "num_tokens": 301380.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 1.412884371727705, |
| "epoch": 4.472289156626506, |
| "grad_norm": 2.841749668121338, |
| "learning_rate": 2.196078431372549e-05, |
| "loss": 1.3348329544067383, |
| "mean_token_accuracy": 0.6968033984303474, |
| "num_tokens": 304449.0, |
| "step": 465 |
| }, |
| { |
| "entropy": 1.4457854568958282, |
| "epoch": 4.5204819277108435, |
| "grad_norm": 2.5401291847229004, |
| "learning_rate": 2e-05, |
| "loss": 1.3989011764526367, |
| "mean_token_accuracy": 0.6939552336931228, |
| "num_tokens": 307631.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 1.3890875995159149, |
| "epoch": 4.5686746987951805, |
| "grad_norm": 2.5059876441955566, |
| "learning_rate": 1.803921568627451e-05, |
| "loss": 1.3057866096496582, |
| "mean_token_accuracy": 0.7126045942306518, |
| "num_tokens": 310718.0, |
| "step": 475 |
| }, |
| { |
| "entropy": 1.4398678675293923, |
| "epoch": 4.6168674698795185, |
| "grad_norm": 2.388061285018921, |
| "learning_rate": 1.607843137254902e-05, |
| "loss": 1.3806493759155274, |
| "mean_token_accuracy": 0.7140536062419415, |
| "num_tokens": 313936.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 1.4656711727380753, |
| "epoch": 4.6650602409638555, |
| "grad_norm": 2.4591257572174072, |
| "learning_rate": 1.411764705882353e-05, |
| "loss": 1.328711700439453, |
| "mean_token_accuracy": 0.705501814186573, |
| "num_tokens": 317225.0, |
| "step": 485 |
| }, |
| { |
| "entropy": 1.4785845473408699, |
| "epoch": 4.713253012048193, |
| "grad_norm": 2.5279247760772705, |
| "learning_rate": 1.215686274509804e-05, |
| "loss": 1.5198940277099608, |
| "mean_token_accuracy": 0.6828307613730431, |
| "num_tokens": 320998.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 1.3545206658542157, |
| "epoch": 4.76144578313253, |
| "grad_norm": 2.389832019805908, |
| "learning_rate": 1.0196078431372549e-05, |
| "loss": 1.3498905181884766, |
| "mean_token_accuracy": 0.7124156683683396, |
| "num_tokens": 324049.0, |
| "step": 495 |
| }, |
| { |
| "entropy": 1.4337334975600242, |
| "epoch": 4.809638554216868, |
| "grad_norm": 2.5963311195373535, |
| "learning_rate": 8.23529411764706e-06, |
| "loss": 1.4652009963989259, |
| "mean_token_accuracy": 0.6918993666768074, |
| "num_tokens": 327578.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 1.3732572302222252, |
| "epoch": 4.857831325301205, |
| "grad_norm": 2.731126070022583, |
| "learning_rate": 6.274509803921569e-06, |
| "loss": 1.255812168121338, |
| "mean_token_accuracy": 0.7217151954770088, |
| "num_tokens": 330740.0, |
| "step": 505 |
| }, |
| { |
| "entropy": 1.3217465229332448, |
| "epoch": 4.906024096385542, |
| "grad_norm": 2.517321825027466, |
| "learning_rate": 4.313725490196079e-06, |
| "loss": 1.262193489074707, |
| "mean_token_accuracy": 0.7306308448314667, |
| "num_tokens": 334020.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 1.4316355839371682, |
| "epoch": 4.95421686746988, |
| "grad_norm": 2.5220019817352295, |
| "learning_rate": 2.3529411764705885e-06, |
| "loss": 1.3470137596130372, |
| "mean_token_accuracy": 0.7120753511786461, |
| "num_tokens": 336979.0, |
| "step": 515 |
| }, |
| { |
| "entropy": 1.4011002747636092, |
| "epoch": 5.0, |
| "grad_norm": 2.894545555114746, |
| "learning_rate": 3.921568627450981e-07, |
| "loss": 1.3720563888549804, |
| "mean_token_accuracy": 0.7007956771474135, |
| "num_tokens": 340005.0, |
| "step": 520 |
| }, |
| { |
| "epoch": 5.0, |
| "eval_entropy": 1.568474531173706, |
| "eval_loss": 1.7007105350494385, |
| "eval_mean_token_accuracy": 0.6262180598882529, |
| "eval_num_tokens": 340005.0, |
| "eval_runtime": 31.0476, |
| "eval_samples_per_second": 6.699, |
| "eval_steps_per_second": 0.837, |
| "step": 520 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 520, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 5, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 2115032078315520.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
