| { | |
| "best_global_step": 100, | |
| "best_metric": 0.04428481683135033, | |
| "best_model_checkpoint": "runs/dpo_run_14b_v1/checkpoint-100", | |
| "epoch": 0.11678832116788321, | |
| "eval_steps": 25, | |
| "global_step": 100, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0023357664233576644, | |
| "grad_norm": 1.242694616317749, | |
| "learning_rate": 1.9379844961240311e-07, | |
| "logits/chosen": 5.179401397705078, | |
| "logits/rejected": 5.192930698394775, | |
| "logps/chosen": -368.911865234375, | |
| "logps/rejected": -398.83880615234375, | |
| "loss": 0.6931473016738892, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.004671532846715329, | |
| "grad_norm": 1.392399787902832, | |
| "learning_rate": 5.813953488372093e-07, | |
| "logits/chosen": 5.403897762298584, | |
| "logits/rejected": 5.4565606117248535, | |
| "logps/chosen": -338.43792724609375, | |
| "logps/rejected": -367.03057861328125, | |
| "loss": 0.6949559450149536, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.004504585638642311, | |
| "rewards/margins": -0.003222561441361904, | |
| "rewards/rejected": 0.007727146148681641, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.0070072992700729924, | |
| "grad_norm": 1.066603183746338, | |
| "learning_rate": 9.689922480620155e-07, | |
| "logits/chosen": 5.291868209838867, | |
| "logits/rejected": 5.328356742858887, | |
| "logps/chosen": -362.3431701660156, | |
| "logps/rejected": -387.5829772949219, | |
| "loss": 0.689236581325531, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0034066196531057358, | |
| "rewards/margins": 0.008255671709775925, | |
| "rewards/rejected": -0.01166229322552681, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.009343065693430658, | |
| "grad_norm": 1.0005714893341064, | |
| "learning_rate": 1.3565891472868218e-06, | |
| "logits/chosen": 5.323437690734863, | |
| "logits/rejected": 5.410858631134033, | |
| "logps/chosen": -379.9283447265625, | |
| "logps/rejected": -389.0852355957031, | |
| "loss": 0.6943775415420532, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.014657974243164062, | |
| "rewards/margins": -0.0012350091710686684, | |
| "rewards/rejected": 0.015892982482910156, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.01167883211678832, | |
| "grad_norm": 1.2461222410202026, | |
| "learning_rate": 1.744186046511628e-06, | |
| "logits/chosen": 5.435908317565918, | |
| "logits/rejected": 5.494542121887207, | |
| "logps/chosen": -363.2003479003906, | |
| "logps/rejected": -389.67376708984375, | |
| "loss": 0.693260908126831, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.028497030958533287, | |
| "rewards/margins": 0.00012636138126254082, | |
| "rewards/rejected": -0.028623390942811966, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.014014598540145985, | |
| "grad_norm": 1.4030137062072754, | |
| "learning_rate": 2.131782945736434e-06, | |
| "logits/chosen": 5.3550801277160645, | |
| "logits/rejected": 5.375768661499023, | |
| "logps/chosen": -370.96429443359375, | |
| "logps/rejected": -402.4786071777344, | |
| "loss": 0.6882913112640381, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.01622028276324272, | |
| "rewards/margins": 0.010086631402373314, | |
| "rewards/rejected": 0.006133650429546833, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.01635036496350365, | |
| "grad_norm": 1.1157702207565308, | |
| "learning_rate": 2.5193798449612402e-06, | |
| "logits/chosen": 5.515308380126953, | |
| "logits/rejected": 5.561104774475098, | |
| "logps/chosen": -336.7254333496094, | |
| "logps/rejected": -357.52203369140625, | |
| "loss": 0.6896716356277466, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.017319394275546074, | |
| "rewards/margins": 0.007328510750085115, | |
| "rewards/rejected": -0.024647902697324753, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.018686131386861315, | |
| "grad_norm": 0.9470655918121338, | |
| "learning_rate": 2.9069767441860468e-06, | |
| "logits/chosen": 5.553088665008545, | |
| "logits/rejected": 5.582851886749268, | |
| "logps/chosen": -415.6842041015625, | |
| "logps/rejected": -441.1054992675781, | |
| "loss": 0.6904245018959045, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.03270244598388672, | |
| "rewards/margins": 0.005826758686453104, | |
| "rewards/rejected": 0.026875685900449753, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.021021897810218976, | |
| "grad_norm": 1.4397331476211548, | |
| "learning_rate": 3.2945736434108533e-06, | |
| "logits/chosen": 5.440742015838623, | |
| "logits/rejected": 5.489529132843018, | |
| "logps/chosen": -392.46221923828125, | |
| "logps/rejected": -420.1712341308594, | |
| "loss": 0.683630108833313, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.011020278558135033, | |
| "rewards/margins": 0.01951923407614231, | |
| "rewards/rejected": -0.008498954586684704, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.02335766423357664, | |
| "grad_norm": 1.5941083431243896, | |
| "learning_rate": 3.6821705426356594e-06, | |
| "logits/chosen": 5.318347930908203, | |
| "logits/rejected": 5.397945404052734, | |
| "logps/chosen": -345.2221374511719, | |
| "logps/rejected": -365.9537048339844, | |
| "loss": 0.6902388334274292, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.006536484230309725, | |
| "rewards/margins": 0.006013393402099609, | |
| "rewards/rejected": 0.0005230908282101154, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.025693430656934305, | |
| "grad_norm": 1.1363905668258667, | |
| "learning_rate": 4.0697674418604655e-06, | |
| "logits/chosen": 5.632981300354004, | |
| "logits/rejected": 5.7265520095825195, | |
| "logps/chosen": -347.9439697265625, | |
| "logps/rejected": -370.65777587890625, | |
| "loss": 0.691262423992157, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.011908342130482197, | |
| "rewards/margins": 0.004538153763860464, | |
| "rewards/rejected": 0.007370188366621733, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.02802919708029197, | |
| "grad_norm": 1.0684627294540405, | |
| "learning_rate": 4.457364341085272e-06, | |
| "logits/chosen": 5.35699987411499, | |
| "logits/rejected": 5.405580520629883, | |
| "logps/chosen": -347.1539001464844, | |
| "logps/rejected": -377.6044921875, | |
| "loss": 0.6769475936889648, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.01244144607335329, | |
| "rewards/margins": 0.03289356082677841, | |
| "rewards/rejected": -0.020452119410037994, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.029197080291970802, | |
| "eval_logits/chosen": 5.295141220092773, | |
| "eval_logits/rejected": 5.345211029052734, | |
| "eval_logps/chosen": -370.1607666015625, | |
| "eval_logps/rejected": -395.7251892089844, | |
| "eval_loss": 0.6836819648742676, | |
| "eval_rewards/accuracies": 0.665354311466217, | |
| "eval_rewards/chosen": 0.024636391550302505, | |
| "eval_rewards/margins": 0.019555427134037018, | |
| "eval_rewards/rejected": 0.005080964416265488, | |
| "eval_runtime": 454.4375, | |
| "eval_samples_per_second": 1.677, | |
| "eval_steps_per_second": 1.677, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.030364963503649634, | |
| "grad_norm": 1.592353105545044, | |
| "learning_rate": 4.844961240310078e-06, | |
| "logits/chosen": 5.157042026519775, | |
| "logits/rejected": 5.244912147521973, | |
| "logps/chosen": -387.54876708984375, | |
| "logps/rejected": -412.0630187988281, | |
| "loss": 0.6849788427352905, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.026385309174656868, | |
| "rewards/margins": 0.016966437920928, | |
| "rewards/rejected": 0.009418869391083717, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.0327007299270073, | |
| "grad_norm": 1.3181558847427368, | |
| "learning_rate": 5.232558139534884e-06, | |
| "logits/chosen": 5.545513153076172, | |
| "logits/rejected": 5.54400110244751, | |
| "logps/chosen": -360.41650390625, | |
| "logps/rejected": -391.2162170410156, | |
| "loss": 0.675189733505249, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 0.045946408063173294, | |
| "rewards/margins": 0.03675585240125656, | |
| "rewards/rejected": 0.009190557524561882, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.035036496350364967, | |
| "grad_norm": 1.443650722503662, | |
| "learning_rate": 5.620155038759691e-06, | |
| "logits/chosen": 5.136168003082275, | |
| "logits/rejected": 5.239327907562256, | |
| "logps/chosen": -378.6293640136719, | |
| "logps/rejected": -405.3665466308594, | |
| "loss": 0.6752142310142517, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 0.04194517061114311, | |
| "rewards/margins": 0.03668833151459694, | |
| "rewards/rejected": 0.005256845150142908, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.03737226277372263, | |
| "grad_norm": 1.379568338394165, | |
| "learning_rate": 6.007751937984497e-06, | |
| "logits/chosen": 5.411487579345703, | |
| "logits/rejected": 5.427243232727051, | |
| "logps/chosen": -358.5367736816406, | |
| "logps/rejected": -382.4181213378906, | |
| "loss": 0.6700581312179565, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.06658173352479935, | |
| "rewards/margins": 0.047193337231874466, | |
| "rewards/rejected": 0.019388392567634583, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.039708029197080295, | |
| "grad_norm": 1.3260451555252075, | |
| "learning_rate": 6.395348837209303e-06, | |
| "logits/chosen": 5.207217216491699, | |
| "logits/rejected": 5.254848480224609, | |
| "logps/chosen": -326.9423828125, | |
| "logps/rejected": -346.52081298828125, | |
| "loss": 0.6610866785049438, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.07038869708776474, | |
| "rewards/margins": 0.06587495654821396, | |
| "rewards/rejected": 0.0045137410052120686, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.04204379562043795, | |
| "grad_norm": 1.5776340961456299, | |
| "learning_rate": 6.782945736434108e-06, | |
| "logits/chosen": 5.550538063049316, | |
| "logits/rejected": 5.6374335289001465, | |
| "logps/chosen": -359.9613952636719, | |
| "logps/rejected": -384.31683349609375, | |
| "loss": 0.6281551718711853, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.11738375574350357, | |
| "rewards/margins": 0.1363767683506012, | |
| "rewards/rejected": -0.018992995843291283, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.04437956204379562, | |
| "grad_norm": 1.8589071035385132, | |
| "learning_rate": 7.170542635658915e-06, | |
| "logits/chosen": 5.39143180847168, | |
| "logits/rejected": 5.412029266357422, | |
| "logps/chosen": -325.8544616699219, | |
| "logps/rejected": -351.9772644042969, | |
| "loss": 0.6270830631256104, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.1617884635925293, | |
| "rewards/margins": 0.1388537436723709, | |
| "rewards/rejected": 0.022934721782803535, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.04671532846715328, | |
| "grad_norm": 1.3231571912765503, | |
| "learning_rate": 7.558139534883721e-06, | |
| "logits/chosen": 5.189720153808594, | |
| "logits/rejected": 5.203127384185791, | |
| "logps/chosen": -343.3839111328125, | |
| "logps/rejected": -374.7848205566406, | |
| "loss": 0.641180157661438, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.15248623490333557, | |
| "rewards/margins": 0.11158552765846252, | |
| "rewards/rejected": 0.04090070724487305, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.049051094890510946, | |
| "grad_norm": 2.5331315994262695, | |
| "learning_rate": 7.945736434108528e-06, | |
| "logits/chosen": 5.420182228088379, | |
| "logits/rejected": 5.45302677154541, | |
| "logps/chosen": -341.813720703125, | |
| "logps/rejected": -372.44952392578125, | |
| "loss": 0.6093671321868896, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.2898235321044922, | |
| "rewards/margins": 0.18158456683158875, | |
| "rewards/rejected": 0.10823898762464523, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.05138686131386861, | |
| "grad_norm": 1.5247384309768677, | |
| "learning_rate": 8.333333333333334e-06, | |
| "logits/chosen": 5.383636951446533, | |
| "logits/rejected": 5.397551536560059, | |
| "logps/chosen": -354.49627685546875, | |
| "logps/rejected": -376.88818359375, | |
| "loss": 0.5815833210945129, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 0.32459571957588196, | |
| "rewards/margins": 0.2510552406311035, | |
| "rewards/rejected": 0.07354050129652023, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.053722627737226275, | |
| "grad_norm": 2.0814144611358643, | |
| "learning_rate": 8.72093023255814e-06, | |
| "logits/chosen": 5.269731044769287, | |
| "logits/rejected": 5.287116050720215, | |
| "logps/chosen": -331.1025390625, | |
| "logps/rejected": -362.90118408203125, | |
| "loss": 0.5269681215286255, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.6465227603912354, | |
| "rewards/margins": 0.37582656741142273, | |
| "rewards/rejected": 0.27069616317749023, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.05605839416058394, | |
| "grad_norm": 1.769063115119934, | |
| "learning_rate": 9.108527131782946e-06, | |
| "logits/chosen": 5.472540855407715, | |
| "logits/rejected": 5.465417861938477, | |
| "logps/chosen": -369.40283203125, | |
| "logps/rejected": -400.18438720703125, | |
| "loss": 0.5066201686859131, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.6377636194229126, | |
| "rewards/margins": 0.42650213837623596, | |
| "rewards/rejected": 0.21126146614551544, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.058394160583941604, | |
| "grad_norm": 2.84169602394104, | |
| "learning_rate": 9.496124031007753e-06, | |
| "logits/chosen": 5.050387382507324, | |
| "logits/rejected": 5.112288951873779, | |
| "logps/chosen": -363.4556579589844, | |
| "logps/rejected": -397.8169860839844, | |
| "loss": 0.529259979724884, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.7923164367675781, | |
| "rewards/margins": 0.3787059783935547, | |
| "rewards/rejected": 0.4136104881763458, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.058394160583941604, | |
| "eval_logits/chosen": 5.22359037399292, | |
| "eval_logits/rejected": 5.286833763122559, | |
| "eval_logps/chosen": -361.462890625, | |
| "eval_logps/rejected": -392.5708312988281, | |
| "eval_loss": 0.4610801041126251, | |
| "eval_rewards/accuracies": 0.9619422554969788, | |
| "eval_rewards/chosen": 0.8944254517555237, | |
| "eval_rewards/margins": 0.5739086270332336, | |
| "eval_rewards/rejected": 0.3205168545246124, | |
| "eval_runtime": 454.5598, | |
| "eval_samples_per_second": 1.676, | |
| "eval_steps_per_second": 1.676, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.06072992700729927, | |
| "grad_norm": 1.6907895803451538, | |
| "learning_rate": 9.883720930232558e-06, | |
| "logits/chosen": 5.486469268798828, | |
| "logits/rejected": 5.541717529296875, | |
| "logps/chosen": -343.4534606933594, | |
| "logps/rejected": -379.39508056640625, | |
| "loss": 0.44602835178375244, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.9869746565818787, | |
| "rewards/margins": 0.6056646108627319, | |
| "rewards/rejected": 0.3813100755214691, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.06306569343065693, | |
| "grad_norm": 1.9458682537078857, | |
| "learning_rate": 1.0271317829457365e-05, | |
| "logits/chosen": 5.169528961181641, | |
| "logits/rejected": 5.2688751220703125, | |
| "logps/chosen": -379.5437316894531, | |
| "logps/rejected": -401.5587463378906, | |
| "loss": 0.43609702587127686, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.7794930934906006, | |
| "rewards/margins": 0.6265671253204346, | |
| "rewards/rejected": 0.15292587876319885, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.0654014598540146, | |
| "grad_norm": 2.1266520023345947, | |
| "learning_rate": 1.065891472868217e-05, | |
| "logits/chosen": 5.097426414489746, | |
| "logits/rejected": 5.15327262878418, | |
| "logps/chosen": -378.0788269042969, | |
| "logps/rejected": -413.27392578125, | |
| "loss": 0.3928414583206177, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 1.274291753768921, | |
| "rewards/margins": 0.7864217758178711, | |
| "rewards/rejected": 0.4878700375556946, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.06773722627737226, | |
| "grad_norm": 1.5381489992141724, | |
| "learning_rate": 1.1046511627906977e-05, | |
| "logits/chosen": 5.138954162597656, | |
| "logits/rejected": 5.20254373550415, | |
| "logps/chosen": -372.93438720703125, | |
| "logps/rejected": -401.8287658691406, | |
| "loss": 0.35855019092559814, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 1.2897911071777344, | |
| "rewards/margins": 0.9354276061058044, | |
| "rewards/rejected": 0.35436347126960754, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.07007299270072993, | |
| "grad_norm": 2.358330726623535, | |
| "learning_rate": 1.1434108527131783e-05, | |
| "logits/chosen": 5.071888446807861, | |
| "logits/rejected": 5.187964916229248, | |
| "logps/chosen": -360.984619140625, | |
| "logps/rejected": -392.3192138671875, | |
| "loss": 0.42801612615585327, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 1.3823509216308594, | |
| "rewards/margins": 0.729066014289856, | |
| "rewards/rejected": 0.6532848477363586, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.07240875912408759, | |
| "grad_norm": 2.177586317062378, | |
| "learning_rate": 1.182170542635659e-05, | |
| "logits/chosen": 5.264093399047852, | |
| "logits/rejected": 5.310842990875244, | |
| "logps/chosen": -364.808349609375, | |
| "logps/rejected": -401.0321044921875, | |
| "loss": 0.31365492939949036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6637591123580933, | |
| "rewards/margins": 1.0887457132339478, | |
| "rewards/rejected": 0.5750135183334351, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.07474452554744526, | |
| "grad_norm": 1.697789192199707, | |
| "learning_rate": 1.2209302325581395e-05, | |
| "logits/chosen": 5.191982269287109, | |
| "logits/rejected": 5.261416912078857, | |
| "logps/chosen": -359.8249816894531, | |
| "logps/rejected": -397.2122497558594, | |
| "loss": 0.3037749230861664, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.6470392942428589, | |
| "rewards/margins": 1.114844799041748, | |
| "rewards/rejected": 0.5321945548057556, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.07708029197080292, | |
| "grad_norm": 1.3219914436340332, | |
| "learning_rate": 1.2596899224806202e-05, | |
| "logits/chosen": 5.293405532836914, | |
| "logits/rejected": 5.3094048500061035, | |
| "logps/chosen": -352.3752136230469, | |
| "logps/rejected": -392.6779479980469, | |
| "loss": 0.25026455521583557, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5671364068984985, | |
| "rewards/margins": 1.4098074436187744, | |
| "rewards/rejected": 0.15732917189598083, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.07941605839416059, | |
| "grad_norm": 1.8173967599868774, | |
| "learning_rate": 1.2984496124031009e-05, | |
| "logits/chosen": 5.025746822357178, | |
| "logits/rejected": 5.114965438842773, | |
| "logps/chosen": -319.99700927734375, | |
| "logps/rejected": -364.115234375, | |
| "loss": 0.3108353912830353, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 1.4788665771484375, | |
| "rewards/margins": 1.2637410163879395, | |
| "rewards/rejected": 0.2151254564523697, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.08175182481751825, | |
| "grad_norm": 1.0658400058746338, | |
| "learning_rate": 1.3372093023255814e-05, | |
| "logits/chosen": 4.945235729217529, | |
| "logits/rejected": 4.959147930145264, | |
| "logps/chosen": -383.84033203125, | |
| "logps/rejected": -431.7752685546875, | |
| "loss": 0.22991834580898285, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.3950352668762207, | |
| "rewards/margins": 1.4965243339538574, | |
| "rewards/rejected": -0.1014888733625412, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.0840875912408759, | |
| "grad_norm": 1.0350896120071411, | |
| "learning_rate": 1.375968992248062e-05, | |
| "logits/chosen": 5.00426721572876, | |
| "logits/rejected": 5.120238780975342, | |
| "logps/chosen": -350.9471435546875, | |
| "logps/rejected": -382.6837158203125, | |
| "loss": 0.22603684663772583, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.2978975772857666, | |
| "rewards/margins": 1.644275426864624, | |
| "rewards/rejected": -0.34637776017189026, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.08642335766423358, | |
| "grad_norm": 1.1595423221588135, | |
| "learning_rate": 1.4147286821705426e-05, | |
| "logits/chosen": 4.890130043029785, | |
| "logits/rejected": 4.9504714012146, | |
| "logps/chosen": -352.34967041015625, | |
| "logps/rejected": -399.23028564453125, | |
| "loss": 0.18921935558319092, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.1984589099884033, | |
| "rewards/margins": 1.7495291233062744, | |
| "rewards/rejected": -0.5510700941085815, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.08759124087591241, | |
| "eval_logits/chosen": 4.930174827575684, | |
| "eval_logits/rejected": 5.032296657562256, | |
| "eval_logps/chosen": -359.19647216796875, | |
| "eval_logps/rejected": -405.1120300292969, | |
| "eval_loss": 0.16020436584949493, | |
| "eval_rewards/accuracies": 0.9960629940032959, | |
| "eval_rewards/chosen": 1.1210675239562988, | |
| "eval_rewards/margins": 2.0546727180480957, | |
| "eval_rewards/rejected": -0.9336051344871521, | |
| "eval_runtime": 454.3435, | |
| "eval_samples_per_second": 1.677, | |
| "eval_steps_per_second": 1.677, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.08875912408759123, | |
| "grad_norm": 1.1433167457580566, | |
| "learning_rate": 1.4534883720930233e-05, | |
| "logits/chosen": 5.037275314331055, | |
| "logits/rejected": 5.1315507888793945, | |
| "logps/chosen": -313.110595703125, | |
| "logps/rejected": -356.1000061035156, | |
| "loss": 0.15998858213424683, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.2128857374191284, | |
| "rewards/margins": 2.0945115089416504, | |
| "rewards/rejected": -0.8816256523132324, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.0910948905109489, | |
| "grad_norm": 0.9839214086532593, | |
| "learning_rate": 1.4922480620155039e-05, | |
| "logits/chosen": 4.817085266113281, | |
| "logits/rejected": 4.874035835266113, | |
| "logps/chosen": -366.2629089355469, | |
| "logps/rejected": -405.7989196777344, | |
| "loss": 0.1894684135913849, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.0605502128601074, | |
| "rewards/margins": 1.90762460231781, | |
| "rewards/rejected": -0.8470743894577026, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.09343065693430656, | |
| "grad_norm": 0.9212782979011536, | |
| "learning_rate": 1.5310077519379846e-05, | |
| "logits/chosen": 5.046716690063477, | |
| "logits/rejected": 5.157979965209961, | |
| "logps/chosen": -348.0658264160156, | |
| "logps/rejected": -395.23870849609375, | |
| "loss": 0.15948188304901123, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.676516056060791, | |
| "rewards/margins": 2.167430877685547, | |
| "rewards/rejected": -1.4909145832061768, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.09576642335766423, | |
| "grad_norm": 0.9820688366889954, | |
| "learning_rate": 1.569767441860465e-05, | |
| "logits/chosen": 4.690741539001465, | |
| "logits/rejected": 4.771791458129883, | |
| "logps/chosen": -378.8666076660156, | |
| "logps/rejected": -436.9100036621094, | |
| "loss": 0.12085139006376266, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.8719685077667236, | |
| "rewards/margins": 2.646538257598877, | |
| "rewards/rejected": -1.7745698690414429, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.09810218978102189, | |
| "grad_norm": 0.66785728931427, | |
| "learning_rate": 1.608527131782946e-05, | |
| "logits/chosen": 4.880465984344482, | |
| "logits/rejected": 4.961792945861816, | |
| "logps/chosen": -346.51214599609375, | |
| "logps/rejected": -400.1110534667969, | |
| "loss": 0.08720710873603821, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.1337480545043945, | |
| "rewards/margins": 2.903944253921509, | |
| "rewards/rejected": -1.7701961994171143, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.10043795620437956, | |
| "grad_norm": 0.5760660767555237, | |
| "learning_rate": 1.647286821705426e-05, | |
| "logits/chosen": 4.464397430419922, | |
| "logits/rejected": 4.680055618286133, | |
| "logps/chosen": -341.7489318847656, | |
| "logps/rejected": -398.322021484375, | |
| "loss": 0.07942983508110046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.2459325790405273, | |
| "rewards/margins": 3.0152552127838135, | |
| "rewards/rejected": -1.7693227529525757, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.10277372262773722, | |
| "grad_norm": 1.6020294427871704, | |
| "learning_rate": 1.686046511627907e-05, | |
| "logits/chosen": 4.563863277435303, | |
| "logits/rejected": 4.680974960327148, | |
| "logps/chosen": -344.9147644042969, | |
| "logps/rejected": -395.4453125, | |
| "loss": 0.1258174479007721, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 1.0706769227981567, | |
| "rewards/margins": 3.118717670440674, | |
| "rewards/rejected": -2.0480403900146484, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.10510948905109489, | |
| "grad_norm": 0.46413859724998474, | |
| "learning_rate": 1.7248062015503875e-05, | |
| "logits/chosen": 4.4989237785339355, | |
| "logits/rejected": 4.673248291015625, | |
| "logps/chosen": -326.9678649902344, | |
| "logps/rejected": -388.4164123535156, | |
| "loss": 0.06663060188293457, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4128761291503906, | |
| "rewards/margins": 3.760685920715332, | |
| "rewards/rejected": -2.3478102684020996, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.10744525547445255, | |
| "grad_norm": 0.6699568629264832, | |
| "learning_rate": 1.7635658914728684e-05, | |
| "logits/chosen": 4.7294535636901855, | |
| "logits/rejected": 4.813880920410156, | |
| "logps/chosen": -362.7267150878906, | |
| "logps/rejected": -439.2985534667969, | |
| "loss": 0.04481709748506546, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.477597713470459, | |
| "rewards/margins": 4.37883186340332, | |
| "rewards/rejected": -2.9012341499328613, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.10978102189781022, | |
| "grad_norm": 0.4152977168560028, | |
| "learning_rate": 1.802325581395349e-05, | |
| "logits/chosen": 4.785149574279785, | |
| "logits/rejected": 4.891542434692383, | |
| "logps/chosen": -381.59246826171875, | |
| "logps/rejected": -444.2817687988281, | |
| "loss": 0.05632612109184265, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.71366286277771, | |
| "rewards/margins": 3.4584720134735107, | |
| "rewards/rejected": -2.744809150695801, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.11211678832116788, | |
| "grad_norm": 0.3152717649936676, | |
| "learning_rate": 1.8410852713178295e-05, | |
| "logits/chosen": 4.603940486907959, | |
| "logits/rejected": 4.804995536804199, | |
| "logps/chosen": -356.7286376953125, | |
| "logps/rejected": -414.69635009765625, | |
| "loss": 0.040920041501522064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7566397190093994, | |
| "rewards/margins": 4.020595550537109, | |
| "rewards/rejected": -2.263956069946289, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.11445255474452555, | |
| "grad_norm": 0.37698569893836975, | |
| "learning_rate": 1.8798449612403103e-05, | |
| "logits/chosen": 4.558542728424072, | |
| "logits/rejected": 4.690641403198242, | |
| "logps/chosen": -339.794189453125, | |
| "logps/rejected": -413.8865966796875, | |
| "loss": 0.025794224813580513, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.3867536783218384, | |
| "rewards/margins": 4.6542744636535645, | |
| "rewards/rejected": -3.2675204277038574, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.11678832116788321, | |
| "grad_norm": 0.15023073554039001, | |
| "learning_rate": 1.918604651162791e-05, | |
| "logits/chosen": 4.387497425079346, | |
| "logits/rejected": 4.494588375091553, | |
| "logps/chosen": -346.2568054199219, | |
| "logps/rejected": -418.9315185546875, | |
| "loss": 0.015155203640460968, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.7938623428344727, | |
| "rewards/margins": 4.942529201507568, | |
| "rewards/rejected": -3.1486666202545166, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.11678832116788321, | |
| "eval_logits/chosen": 4.285891056060791, | |
| "eval_logits/rejected": 4.425926208496094, | |
| "eval_logps/chosen": -353.15850830078125, | |
| "eval_logps/rejected": -424.4124755859375, | |
| "eval_loss": 0.04428481683135033, | |
| "eval_rewards/accuracies": 0.9921259880065918, | |
| "eval_rewards/chosen": 1.7248634099960327, | |
| "eval_rewards/margins": 4.588510513305664, | |
| "eval_rewards/rejected": -2.863647222518921, | |
| "eval_runtime": 454.7251, | |
| "eval_samples_per_second": 1.676, | |
| "eval_steps_per_second": 1.676, | |
| "step": 100 | |
| } | |
| ], | |
| "logging_steps": 2, | |
| "max_steps": 2571, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "EarlyStoppingCallback": { | |
| "args": { | |
| "early_stopping_patience": 5, | |
| "early_stopping_threshold": 0.001 | |
| }, | |
| "attributes": { | |
| "early_stopping_patience_counter": 0 | |
| } | |
| }, | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |