| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.0, |
| "eval_steps": 250, |
| "global_step": 558, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.3969754904508591, |
| "epoch": 0.03587443946188341, |
| "grad_norm": 0.62109375, |
| "learning_rate": 5.294117647058824e-05, |
| "loss": 2.051294708251953, |
| "mean_token_accuracy": 0.5748387612402439, |
| "num_tokens": 92395.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.1848952442407608, |
| "epoch": 0.07174887892376682, |
| "grad_norm": 0.34375, |
| "learning_rate": 9.999662790086726e-05, |
| "loss": 1.152494525909424, |
| "mean_token_accuracy": 0.7453485190868377, |
| "num_tokens": 204363.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.6448489271104336, |
| "epoch": 0.10762331838565023, |
| "grad_norm": 0.275390625, |
| "learning_rate": 9.987865218277088e-05, |
| "loss": 0.67830491065979, |
| "mean_token_accuracy": 0.8489482343196869, |
| "num_tokens": 307781.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.5587928161025048, |
| "epoch": 0.14349775784753363, |
| "grad_norm": 0.33203125, |
| "learning_rate": 9.959252607444427e-05, |
| "loss": 0.6177556991577149, |
| "mean_token_accuracy": 0.8661479398608207, |
| "num_tokens": 419690.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.5453228108584881, |
| "epoch": 0.17937219730941703, |
| "grad_norm": 0.2578125, |
| "learning_rate": 9.913921416122937e-05, |
| "loss": 0.6175318241119385, |
| "mean_token_accuracy": 0.863138884305954, |
| "num_tokens": 518165.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.5307354614138603, |
| "epoch": 0.21524663677130046, |
| "grad_norm": 0.2021484375, |
| "learning_rate": 9.852024464350382e-05, |
| "loss": 0.5555857181549072, |
| "mean_token_accuracy": 0.8736034348607064, |
| "num_tokens": 632573.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.5804678507149219, |
| "epoch": 0.25112107623318386, |
| "grad_norm": 0.26953125, |
| "learning_rate": 9.773770418482792e-05, |
| "loss": 0.6112886905670166, |
| "mean_token_accuracy": 0.8588371470570564, |
| "num_tokens": 746882.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.4803255133330822, |
| "epoch": 0.28699551569506726, |
| "grad_norm": 0.1953125, |
| "learning_rate": 9.679423087740625e-05, |
| "loss": 0.5199193000793457, |
| "mean_token_accuracy": 0.875262650847435, |
| "num_tokens": 860710.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.4950253814458847, |
| "epoch": 0.32286995515695066, |
| "grad_norm": 0.2734375, |
| "learning_rate": 9.569300534857875e-05, |
| "loss": 0.5125812530517578, |
| "mean_token_accuracy": 0.876356054842472, |
| "num_tokens": 973776.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.5185210466384887, |
| "epoch": 0.35874439461883406, |
| "grad_norm": 0.25390625, |
| "learning_rate": 9.443774003832332e-05, |
| "loss": 0.5369254112243652, |
| "mean_token_accuracy": 0.8700352162122726, |
| "num_tokens": 1081833.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.47246704772114756, |
| "epoch": 0.39461883408071746, |
| "grad_norm": 0.337890625, |
| "learning_rate": 9.303266668391679e-05, |
| "loss": 0.4889512062072754, |
| "mean_token_accuracy": 0.8794563993811607, |
| "num_tokens": 1195496.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.4534280378371477, |
| "epoch": 0.4304932735426009, |
| "grad_norm": 0.33984375, |
| "learning_rate": 9.148252205394665e-05, |
| "loss": 0.4708422660827637, |
| "mean_token_accuracy": 0.8855773583054543, |
| "num_tokens": 1299697.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.42982515431940554, |
| "epoch": 0.4663677130044843, |
| "grad_norm": 0.275390625, |
| "learning_rate": 8.979253197976633e-05, |
| "loss": 0.44471211433410646, |
| "mean_token_accuracy": 0.8913678646087646, |
| "num_tokens": 1401611.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.42483637742698194, |
| "epoch": 0.5022421524663677, |
| "grad_norm": 0.2060546875, |
| "learning_rate": 8.796839373822721e-05, |
| "loss": 0.43284382820129397, |
| "mean_token_accuracy": 0.8964968830347061, |
| "num_tokens": 1511714.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.38399339467287064, |
| "epoch": 0.5381165919282511, |
| "grad_norm": 0.2236328125, |
| "learning_rate": 8.60162568450785e-05, |
| "loss": 0.416139554977417, |
| "mean_token_accuracy": 0.8991150856018066, |
| "num_tokens": 1613398.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.41539575792849065, |
| "epoch": 0.5739910313901345, |
| "grad_norm": 0.240234375, |
| "learning_rate": 8.394270232378419e-05, |
| "loss": 0.4283764839172363, |
| "mean_token_accuracy": 0.8929989218711853, |
| "num_tokens": 1710736.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.47740858122706414, |
| "epoch": 0.6098654708520179, |
| "grad_norm": 0.263671875, |
| "learning_rate": 8.175472051964552e-05, |
| "loss": 0.4906760692596436, |
| "mean_token_accuracy": 0.8809969320893287, |
| "num_tokens": 1815452.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.4214516717940569, |
| "epoch": 0.6457399103139013, |
| "grad_norm": 0.279296875, |
| "learning_rate": 7.945968753402216e-05, |
| "loss": 0.44007339477539065, |
| "mean_token_accuracy": 0.8928113669157028, |
| "num_tokens": 1917703.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.4015789397060871, |
| "epoch": 0.6816143497757847, |
| "grad_norm": 0.28125, |
| "learning_rate": 7.70653403580964e-05, |
| "loss": 0.4222086429595947, |
| "mean_token_accuracy": 0.8981553256511688, |
| "num_tokens": 2025309.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.4304059199988842, |
| "epoch": 0.7174887892376681, |
| "grad_norm": 0.283203125, |
| "learning_rate": 7.457975079000954e-05, |
| "loss": 0.4446873188018799, |
| "mean_token_accuracy": 0.8913767769932747, |
| "num_tokens": 2126377.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.40952816382050516, |
| "epoch": 0.7533632286995515, |
| "grad_norm": 0.259765625, |
| "learning_rate": 7.201129822330041e-05, |
| "loss": 0.4252801418304443, |
| "mean_token_accuracy": 0.8945473477244377, |
| "num_tokens": 2239100.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.414412359893322, |
| "epoch": 0.7892376681614349, |
| "grad_norm": 0.28515625, |
| "learning_rate": 6.936864139838158e-05, |
| "loss": 0.4217662811279297, |
| "mean_token_accuracy": 0.8939862757921219, |
| "num_tokens": 2335750.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.45669897869229314, |
| "epoch": 0.8251121076233184, |
| "grad_norm": 0.294921875, |
| "learning_rate": 6.666068921228433e-05, |
| "loss": 0.46158409118652344, |
| "mean_token_accuracy": 0.8881383046507836, |
| "num_tokens": 2451761.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.4027619417756796, |
| "epoch": 0.8609865470852018, |
| "grad_norm": 0.314453125, |
| "learning_rate": 6.389657068507819e-05, |
| "loss": 0.41291356086730957, |
| "mean_token_accuracy": 0.8973603308200836, |
| "num_tokens": 2553271.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.38662399984896184, |
| "epoch": 0.8968609865470852, |
| "grad_norm": 0.279296875, |
| "learning_rate": 6.108560418421397e-05, |
| "loss": 0.38063457012176516, |
| "mean_token_accuracy": 0.9039406031370163, |
| "num_tokens": 2656981.0, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.8968609865470852, |
| "eval_entropy": 0.4262866288738907, |
| "eval_loss": 0.5014730095863342, |
| "eval_mean_token_accuracy": 0.8819858220519212, |
| "eval_num_tokens": 2656981.0, |
| "eval_runtime": 278.3421, |
| "eval_samples_per_second": 5.421, |
| "eval_steps_per_second": 1.358, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.38649687841534613, |
| "epoch": 0.9327354260089686, |
| "grad_norm": 0.3046875, |
| "learning_rate": 5.8237266010541046e-05, |
| "loss": 0.4081888675689697, |
| "mean_token_accuracy": 0.8996688678860665, |
| "num_tokens": 2759594.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.3772343125194311, |
| "epoch": 0.968609865470852, |
| "grad_norm": 0.244140625, |
| "learning_rate": 5.536115845190105e-05, |
| "loss": 0.3852355480194092, |
| "mean_token_accuracy": 0.902637392282486, |
| "num_tokens": 2859621.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.42768477018062884, |
| "epoch": 1.0035874439461883, |
| "grad_norm": 0.2373046875, |
| "learning_rate": 5.2466977411995567e-05, |
| "loss": 0.4248982906341553, |
| "mean_token_accuracy": 0.8919965258011451, |
| "num_tokens": 2972688.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.3386215832084417, |
| "epoch": 1.0394618834080718, |
| "grad_norm": 0.30859375, |
| "learning_rate": 4.9564479723657075e-05, |
| "loss": 0.3396793603897095, |
| "mean_token_accuracy": 0.9132053166627884, |
| "num_tokens": 3067583.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.3685107693076134, |
| "epoch": 1.075336322869955, |
| "grad_norm": 0.322265625, |
| "learning_rate": 4.6663450256716226e-05, |
| "loss": 0.3796958923339844, |
| "mean_token_accuracy": 0.9038235887885093, |
| "num_tokens": 3172296.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.35373203866183756, |
| "epoch": 1.1112107623318386, |
| "grad_norm": 0.2392578125, |
| "learning_rate": 4.3773668931351055e-05, |
| "loss": 0.35225601196289064, |
| "mean_token_accuracy": 0.9102007254958153, |
| "num_tokens": 3276803.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.3816000144928694, |
| "epoch": 1.1470852017937219, |
| "grad_norm": 0.328125, |
| "learning_rate": 4.0904877748122436e-05, |
| "loss": 0.38906025886535645, |
| "mean_token_accuracy": 0.9023444890975952, |
| "num_tokens": 3391132.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.3066167201846838, |
| "epoch": 1.1829596412556054, |
| "grad_norm": 0.294921875, |
| "learning_rate": 3.806674794584374e-05, |
| "loss": 0.30766735076904295, |
| "mean_token_accuracy": 0.921451187133789, |
| "num_tokens": 3497920.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.3733608294278383, |
| "epoch": 1.2188340807174889, |
| "grad_norm": 0.365234375, |
| "learning_rate": 3.5268847398001766e-05, |
| "loss": 0.39584877490997317, |
| "mean_token_accuracy": 0.9041571974754333, |
| "num_tokens": 3599625.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.3333755239844322, |
| "epoch": 1.2547085201793722, |
| "grad_norm": 0.30078125, |
| "learning_rate": 3.252060835764181e-05, |
| "loss": 0.3276608943939209, |
| "mean_token_accuracy": 0.9138269037008285, |
| "num_tokens": 3707539.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.335588826239109, |
| "epoch": 1.2905829596412555, |
| "grad_norm": 0.248046875, |
| "learning_rate": 2.9831295659454838e-05, |
| "loss": 0.3360959529876709, |
| "mean_token_accuracy": 0.9134533792734146, |
| "num_tokens": 3813716.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.32520635724067687, |
| "epoch": 1.326457399103139, |
| "grad_norm": 0.337890625, |
| "learning_rate": 2.7209975486263807e-05, |
| "loss": 0.3224537134170532, |
| "mean_token_accuracy": 0.9162669718265534, |
| "num_tokens": 3924180.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.3270633514970541, |
| "epoch": 1.3623318385650225, |
| "grad_norm": 0.283203125, |
| "learning_rate": 2.466548480520296e-05, |
| "loss": 0.323980975151062, |
| "mean_token_accuracy": 0.9139964297413826, |
| "num_tokens": 4026780.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.3498925618827343, |
| "epoch": 1.3982062780269058, |
| "grad_norm": 0.30078125, |
| "learning_rate": 2.220640157662679e-05, |
| "loss": 0.37245047092437744, |
| "mean_token_accuracy": 0.9063089922070503, |
| "num_tokens": 4130194.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.33601379320025443, |
| "epoch": 1.4340807174887893, |
| "grad_norm": 0.353515625, |
| "learning_rate": 1.9841015836179734e-05, |
| "loss": 0.32559800148010254, |
| "mean_token_accuracy": 0.9136791557073594, |
| "num_tokens": 4236882.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.322609855979681, |
| "epoch": 1.4699551569506726, |
| "grad_norm": 0.36328125, |
| "learning_rate": 1.7577301747514922e-05, |
| "loss": 0.33389127254486084, |
| "mean_token_accuracy": 0.9131381779909133, |
| "num_tokens": 4338348.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.3596556928008795, |
| "epoch": 1.505829596412556, |
| "grad_norm": 0.369140625, |
| "learning_rate": 1.5422890719877e-05, |
| "loss": 0.36695308685302735, |
| "mean_token_accuracy": 0.9074400275945663, |
| "num_tokens": 4439178.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.3173448100686073, |
| "epoch": 1.5417040358744396, |
| "grad_norm": 0.34375, |
| "learning_rate": 1.338504568117518e-05, |
| "loss": 0.3288587093353271, |
| "mean_token_accuracy": 0.9189538955688477, |
| "num_tokens": 4548339.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.3539650794118643, |
| "epoch": 1.577578475336323, |
| "grad_norm": 0.3046875, |
| "learning_rate": 1.1470636593276779e-05, |
| "loss": 0.35024569034576414, |
| "mean_token_accuracy": 0.9065869942307472, |
| "num_tokens": 4660945.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.33447692096233367, |
| "epoch": 1.6134529147982062, |
| "grad_norm": 0.30078125, |
| "learning_rate": 9.686117292063501e-06, |
| "loss": 0.3201354742050171, |
| "mean_token_accuracy": 0.9138964533805847, |
| "num_tokens": 4760382.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.3287306733429432, |
| "epoch": 1.6493273542600897, |
| "grad_norm": 0.296875, |
| "learning_rate": 8.037503730327634e-06, |
| "loss": 0.32874948978424073, |
| "mean_token_accuracy": 0.9138829991221428, |
| "num_tokens": 4874370.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.3220908358693123, |
| "epoch": 1.6852017937219732, |
| "grad_norm": 0.279296875, |
| "learning_rate": 6.530353696854791e-06, |
| "loss": 0.3312650442123413, |
| "mean_token_accuracy": 0.9188773602247238, |
| "num_tokens": 4996174.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.34412881508469584, |
| "epoch": 1.7210762331838565, |
| "grad_norm": 0.3046875, |
| "learning_rate": 5.169748080064651e-06, |
| "loss": 0.338838791847229, |
| "mean_token_accuracy": 0.9112714245915413, |
| "num_tokens": 5105558.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.3259685214608908, |
| "epoch": 1.7569506726457398, |
| "grad_norm": 0.3359375, |
| "learning_rate": 3.960273739373044e-06, |
| "loss": 0.3209350824356079, |
| "mean_token_accuracy": 0.9152405560016632, |
| "num_tokens": 5220771.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.3517936758697033, |
| "epoch": 1.7928251121076233, |
| "grad_norm": 0.294921875, |
| "learning_rate": 2.9060080420194648e-06, |
| "loss": 0.35311083793640136, |
| "mean_token_accuracy": 0.9091673374176026, |
| "num_tokens": 5326562.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.7928251121076233, |
| "eval_entropy": 0.3918723243648413, |
| "eval_loss": 0.5272403955459595, |
| "eval_mean_token_accuracy": 0.8795070842144981, |
| "eval_num_tokens": 5326562.0, |
| "eval_runtime": 277.68, |
| "eval_samples_per_second": 5.434, |
| "eval_steps_per_second": 1.361, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.3449239183217287, |
| "epoch": 1.8286995515695068, |
| "grad_norm": 0.2890625, |
| "learning_rate": 2.0105051174893363e-06, |
| "loss": 0.3280923366546631, |
| "mean_token_accuracy": 0.9127265498042106, |
| "num_tokens": 5439381.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.32186402194201946, |
| "epoch": 1.86457399103139, |
| "grad_norm": 0.404296875, |
| "learning_rate": 1.2767838758698912e-06, |
| "loss": 0.32121551036834717, |
| "mean_token_accuracy": 0.9174392357468605, |
| "num_tokens": 5533378.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.3244446087628603, |
| "epoch": 1.9004484304932734, |
| "grad_norm": 0.29296875, |
| "learning_rate": 7.073178305320727e-07, |
| "loss": 0.310935378074646, |
| "mean_token_accuracy": 0.917025238275528, |
| "num_tokens": 5637745.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.3581434443593025, |
| "epoch": 1.936322869955157, |
| "grad_norm": 0.318359375, |
| "learning_rate": 3.0402675944814496e-07, |
| "loss": 0.359324312210083, |
| "mean_token_accuracy": 0.9096325069665909, |
| "num_tokens": 5742787.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.3250444334000349, |
| "epoch": 1.9721973094170404, |
| "grad_norm": 0.30078125, |
| "learning_rate": 6.827023325626237e-08, |
| "loss": 0.3184437990188599, |
| "mean_token_accuracy": 0.9155865594744682, |
| "num_tokens": 5844462.0, |
| "step": 550 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_entropy": 0.3912128670111535, |
| "eval_loss": 0.5274220705032349, |
| "eval_mean_token_accuracy": 0.8794930301330708, |
| "eval_num_tokens": 5924816.0, |
| "eval_runtime": 277.7162, |
| "eval_samples_per_second": 5.434, |
| "eval_steps_per_second": 1.361, |
| "step": 558 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 558, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 250, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 4.960190639222784e+17, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
