| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.967020426794477, |
| "eval_steps": 100, |
| "global_step": 6500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0, |
| "learning_rate": 4.5662100456621e-10, |
| "logits/chosen": -2.4946703910827637, |
| "logits/rejected": -2.335416316986084, |
| "logps/chosen": -85.90689086914062, |
| "logps/rejected": -62.35003662109375, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.0, |
| "learning_rate": 4.5662100456621e-09, |
| "logits/chosen": -2.267062187194824, |
| "logits/rejected": -1.9090423583984375, |
| "logps/chosen": -88.730712890625, |
| "logps/rejected": -68.25267028808594, |
| "loss": 0.6976, |
| "rewards/accuracies": 0.5138888955116272, |
| "rewards/chosen": 0.0035505560226738453, |
| "rewards/margins": 0.019558124244213104, |
| "rewards/rejected": -0.016007568687200546, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 9.1324200913242e-09, |
| "logits/chosen": -2.2504024505615234, |
| "logits/rejected": -1.9175926446914673, |
| "logps/chosen": -94.94258880615234, |
| "logps/rejected": -70.55738067626953, |
| "loss": 0.6949, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.02249746397137642, |
| "rewards/margins": 0.018389523029327393, |
| "rewards/rejected": 0.004107940010726452, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.01, |
| "learning_rate": 1.36986301369863e-08, |
| "logits/chosen": -2.2938480377197266, |
| "logits/rejected": -1.9231021404266357, |
| "logps/chosen": -89.45714569091797, |
| "logps/rejected": -70.13645935058594, |
| "loss": 0.6899, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": 0.025471080094575882, |
| "rewards/margins": 0.030790437012910843, |
| "rewards/rejected": -0.005319356918334961, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 1.82648401826484e-08, |
| "logits/chosen": -2.323638439178467, |
| "logits/rejected": -1.9527565240859985, |
| "logps/chosen": -92.12770080566406, |
| "logps/rejected": -68.82365417480469, |
| "loss": 0.6845, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.022075748071074486, |
| "rewards/margins": 0.03521919250488281, |
| "rewards/rejected": -0.013143444433808327, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.02, |
| "learning_rate": 2.28310502283105e-08, |
| "logits/chosen": -2.302112340927124, |
| "logits/rejected": -1.9624712467193604, |
| "logps/chosen": -89.31111907958984, |
| "logps/rejected": -67.56095123291016, |
| "loss": 0.6716, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.02959294244647026, |
| "rewards/margins": 0.04926164075732231, |
| "rewards/rejected": -0.01966869831085205, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 2.73972602739726e-08, |
| "logits/chosen": -2.3690874576568604, |
| "logits/rejected": -2.003035068511963, |
| "logps/chosen": -94.92839050292969, |
| "logps/rejected": -72.64738464355469, |
| "loss": 0.6588, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.028924476355314255, |
| "rewards/margins": 0.0910693034529686, |
| "rewards/rejected": -0.06214482709765434, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.03, |
| "learning_rate": 3.19634703196347e-08, |
| "logits/chosen": -2.251183032989502, |
| "logits/rejected": -1.8819090127944946, |
| "logps/chosen": -91.9073257446289, |
| "logps/rejected": -69.90731811523438, |
| "loss": 0.6357, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.06686282157897949, |
| "rewards/margins": 0.17646007239818573, |
| "rewards/rejected": -0.10959725081920624, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 3.65296803652968e-08, |
| "logits/chosen": -2.2135210037231445, |
| "logits/rejected": -1.8786967992782593, |
| "logps/chosen": -96.14964294433594, |
| "logps/rejected": -70.20441436767578, |
| "loss": 0.5988, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.11508840322494507, |
| "rewards/margins": 0.2152937352657318, |
| "rewards/rejected": -0.10020533949136734, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.04, |
| "learning_rate": 4.10958904109589e-08, |
| "logits/chosen": -2.2189507484436035, |
| "logits/rejected": -1.9139974117279053, |
| "logps/chosen": -83.40045928955078, |
| "logps/rejected": -64.86124420166016, |
| "loss": 0.5451, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.12101010233163834, |
| "rewards/margins": 0.3227211534976959, |
| "rewards/rejected": -0.2017110288143158, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 4.5662100456621e-08, |
| "logits/chosen": -2.3013663291931152, |
| "logits/rejected": -1.9398345947265625, |
| "logps/chosen": -89.51341247558594, |
| "logps/rejected": -67.42799377441406, |
| "loss": 0.5166, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1514771282672882, |
| "rewards/margins": 0.42826494574546814, |
| "rewards/rejected": -0.27678781747817993, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.05, |
| "eval_logits/chosen": -2.261293411254883, |
| "eval_logits/rejected": -1.9334125518798828, |
| "eval_logps/chosen": -88.47880554199219, |
| "eval_logps/rejected": -67.38658905029297, |
| "eval_loss": 0.5020039081573486, |
| "eval_rewards/accuracies": 0.9497206807136536, |
| "eval_rewards/chosen": 0.17589102685451508, |
| "eval_rewards/margins": 0.4537213146686554, |
| "eval_rewards/rejected": -0.2778303027153015, |
| "eval_runtime": 310.5221, |
| "eval_samples_per_second": 9.217, |
| "eval_steps_per_second": 0.576, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 5.02283105022831e-08, |
| "logits/chosen": -2.3438382148742676, |
| "logits/rejected": -1.9776771068572998, |
| "logps/chosen": -84.6239013671875, |
| "logps/rejected": -64.25120544433594, |
| "loss": 0.4884, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.23512157797813416, |
| "rewards/margins": 0.4826792776584625, |
| "rewards/rejected": -0.24755771458148956, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.05, |
| "learning_rate": 5.47945205479452e-08, |
| "logits/chosen": -2.305763006210327, |
| "logits/rejected": -1.9242515563964844, |
| "logps/chosen": -93.78631591796875, |
| "logps/rejected": -71.03646087646484, |
| "loss": 0.452, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.25619563460350037, |
| "rewards/margins": 0.7088645100593567, |
| "rewards/rejected": -0.45266884565353394, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 5.93607305936073e-08, |
| "logits/chosen": -2.1994524002075195, |
| "logits/rejected": -1.8348830938339233, |
| "logps/chosen": -93.8993911743164, |
| "logps/rejected": -67.64842224121094, |
| "loss": 0.3697, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.37854477763175964, |
| "rewards/margins": 0.9585170745849609, |
| "rewards/rejected": -0.5799722671508789, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 6.39269406392694e-08, |
| "logits/chosen": -2.339900493621826, |
| "logits/rejected": -1.9880163669586182, |
| "logps/chosen": -91.68956756591797, |
| "logps/rejected": -70.57650756835938, |
| "loss": 0.3139, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4103256165981293, |
| "rewards/margins": 1.110944390296936, |
| "rewards/rejected": -0.7006188631057739, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 6.84931506849315e-08, |
| "logits/chosen": -2.3096134662628174, |
| "logits/rejected": -1.9507678747177124, |
| "logps/chosen": -88.14106750488281, |
| "logps/rejected": -69.87593841552734, |
| "loss": 0.2834, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5618699789047241, |
| "rewards/margins": 1.299116849899292, |
| "rewards/rejected": -0.7372468113899231, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.07, |
| "learning_rate": 7.30593607305936e-08, |
| "logits/chosen": -2.2230865955352783, |
| "logits/rejected": -1.8755073547363281, |
| "logps/chosen": -93.29566955566406, |
| "logps/rejected": -71.75144958496094, |
| "loss": 0.2673, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.5255166292190552, |
| "rewards/margins": 1.4965600967407227, |
| "rewards/rejected": -0.9710434079170227, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 7.76255707762557e-08, |
| "logits/chosen": -2.1584527492523193, |
| "logits/rejected": -1.9142663478851318, |
| "logps/chosen": -84.84449768066406, |
| "logps/rejected": -71.8950424194336, |
| "loss": 0.2645, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.43970975279808044, |
| "rewards/margins": 1.3599587678909302, |
| "rewards/rejected": -0.9202489852905273, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.08, |
| "learning_rate": 8.21917808219178e-08, |
| "logits/chosen": -2.317610740661621, |
| "logits/rejected": -1.8754488229751587, |
| "logps/chosen": -94.67909240722656, |
| "logps/rejected": -69.68304443359375, |
| "loss": 0.2408, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.7285944819450378, |
| "rewards/margins": 1.5651540756225586, |
| "rewards/rejected": -0.8365596532821655, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 8.67579908675799e-08, |
| "logits/chosen": -2.3835504055023193, |
| "logits/rejected": -2.005877733230591, |
| "logps/chosen": -89.22205352783203, |
| "logps/rejected": -72.03932189941406, |
| "loss": 0.2142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7577417492866516, |
| "rewards/margins": 1.8872630596160889, |
| "rewards/rejected": -1.1295212507247925, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.09, |
| "learning_rate": 9.1324200913242e-08, |
| "logits/chosen": -2.2697577476501465, |
| "logits/rejected": -1.892371416091919, |
| "logps/chosen": -89.17048645019531, |
| "logps/rejected": -69.54945373535156, |
| "loss": 0.1947, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7742541432380676, |
| "rewards/margins": 1.9032939672470093, |
| "rewards/rejected": -1.1290397644042969, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.09, |
| "eval_logits/chosen": -2.2516348361968994, |
| "eval_logits/rejected": -1.9177496433258057, |
| "eval_logps/chosen": -87.51004028320312, |
| "eval_logps/rejected": -69.3072738647461, |
| "eval_loss": 0.19914411008358002, |
| "eval_rewards/accuracies": 0.9748603105545044, |
| "eval_rewards/chosen": 0.6602736115455627, |
| "eval_rewards/margins": 1.898452877998352, |
| "eval_rewards/rejected": -1.2381792068481445, |
| "eval_runtime": 222.9167, |
| "eval_samples_per_second": 12.839, |
| "eval_steps_per_second": 0.803, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 9.58904109589041e-08, |
| "logits/chosen": -2.25944185256958, |
| "logits/rejected": -1.9700183868408203, |
| "logps/chosen": -84.00562286376953, |
| "logps/rejected": -68.82413482666016, |
| "loss": 0.1838, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.6912348866462708, |
| "rewards/margins": 1.9548532962799072, |
| "rewards/rejected": -1.2636187076568604, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 1.004566210045662e-07, |
| "logits/chosen": -2.2063567638397217, |
| "logits/rejected": -1.9247395992279053, |
| "logps/chosen": -83.2778091430664, |
| "logps/rejected": -68.1301498413086, |
| "loss": 0.1654, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.7392381429672241, |
| "rewards/margins": 2.103783130645752, |
| "rewards/rejected": -1.3645451068878174, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.1, |
| "learning_rate": 1.050228310502283e-07, |
| "logits/chosen": -2.196837902069092, |
| "logits/rejected": -1.8360687494277954, |
| "logps/chosen": -90.70558166503906, |
| "logps/rejected": -71.97917175292969, |
| "loss": 0.138, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.6917696595191956, |
| "rewards/margins": 2.567314386367798, |
| "rewards/rejected": -1.875544786453247, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 1.095890410958904e-07, |
| "logits/chosen": -2.269440174102783, |
| "logits/rejected": -1.8552277088165283, |
| "logps/chosen": -91.484130859375, |
| "logps/rejected": -67.85257720947266, |
| "loss": 0.1148, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.0015159845352173, |
| "rewards/margins": 2.8821816444396973, |
| "rewards/rejected": -1.8806654214859009, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.11, |
| "learning_rate": 1.141552511415525e-07, |
| "logits/chosen": -2.2505125999450684, |
| "logits/rejected": -1.8530277013778687, |
| "logps/chosen": -91.98687744140625, |
| "logps/rejected": -76.97599792480469, |
| "loss": 0.0979, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.9848436117172241, |
| "rewards/margins": 3.093383550643921, |
| "rewards/rejected": -2.1085400581359863, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 1.187214611872146e-07, |
| "logits/chosen": -2.3471381664276123, |
| "logits/rejected": -1.9955825805664062, |
| "logps/chosen": -89.78094482421875, |
| "logps/rejected": -75.41387939453125, |
| "loss": 0.0891, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.1348358392715454, |
| "rewards/margins": 3.3899853229522705, |
| "rewards/rejected": -2.2551493644714355, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.12, |
| "learning_rate": 1.232876712328767e-07, |
| "logits/chosen": -2.2680814266204834, |
| "logits/rejected": -1.8932344913482666, |
| "logps/chosen": -90.12066650390625, |
| "logps/rejected": -74.62397766113281, |
| "loss": 0.0802, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1657724380493164, |
| "rewards/margins": 3.604767322540283, |
| "rewards/rejected": -2.438995122909546, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 1.278538812785388e-07, |
| "logits/chosen": -2.276060104370117, |
| "logits/rejected": -1.9517319202423096, |
| "logps/chosen": -91.58979797363281, |
| "logps/rejected": -80.03755187988281, |
| "loss": 0.067, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0883013010025024, |
| "rewards/margins": 4.193854808807373, |
| "rewards/rejected": -3.1055538654327393, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 1.324200913242009e-07, |
| "logits/chosen": -2.288156509399414, |
| "logits/rejected": -1.8913819789886475, |
| "logps/chosen": -92.2828598022461, |
| "logps/rejected": -77.56204986572266, |
| "loss": 0.0674, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2855546474456787, |
| "rewards/margins": 3.8462796211242676, |
| "rewards/rejected": -2.560725212097168, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 1.36986301369863e-07, |
| "logits/chosen": -2.221187114715576, |
| "logits/rejected": -1.8750922679901123, |
| "logps/chosen": -84.05517578125, |
| "logps/rejected": -73.2014389038086, |
| "loss": 0.0662, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3618335723876953, |
| "rewards/margins": 4.3457865715026855, |
| "rewards/rejected": -2.9839529991149902, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.14, |
| "eval_logits/chosen": -2.245551586151123, |
| "eval_logits/rejected": -1.9041495323181152, |
| "eval_logps/chosen": -86.4066162109375, |
| "eval_logps/rejected": -73.35151672363281, |
| "eval_loss": 0.06390678137540817, |
| "eval_rewards/accuracies": 0.9804469347000122, |
| "eval_rewards/chosen": 1.2119877338409424, |
| "eval_rewards/margins": 4.47228479385376, |
| "eval_rewards/rejected": -3.260296106338501, |
| "eval_runtime": 197.1717, |
| "eval_samples_per_second": 14.515, |
| "eval_steps_per_second": 0.908, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.14, |
| "learning_rate": 1.415525114155251e-07, |
| "logits/chosen": -2.24013090133667, |
| "logits/rejected": -1.8544371128082275, |
| "logps/chosen": -88.1822280883789, |
| "logps/rejected": -72.43907928466797, |
| "loss": 0.0602, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3825366497039795, |
| "rewards/margins": 4.868518829345703, |
| "rewards/rejected": -3.4859824180603027, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 1.461187214611872e-07, |
| "logits/chosen": -2.2278833389282227, |
| "logits/rejected": -1.8756189346313477, |
| "logps/chosen": -93.53022003173828, |
| "logps/rejected": -76.30734252929688, |
| "loss": 0.0517, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2675020694732666, |
| "rewards/margins": 4.621975898742676, |
| "rewards/rejected": -3.35447359085083, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.15, |
| "learning_rate": 1.506849315068493e-07, |
| "logits/chosen": -2.2458999156951904, |
| "logits/rejected": -1.9483330249786377, |
| "logps/chosen": -84.49108123779297, |
| "logps/rejected": -74.85401153564453, |
| "loss": 0.0488, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4710012674331665, |
| "rewards/margins": 5.068787574768066, |
| "rewards/rejected": -3.5977866649627686, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 1.552511415525114e-07, |
| "logits/chosen": -2.2143704891204834, |
| "logits/rejected": -1.8886018991470337, |
| "logps/chosen": -85.12834167480469, |
| "logps/rejected": -73.5596694946289, |
| "loss": 0.0535, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3029371500015259, |
| "rewards/margins": 5.119471073150635, |
| "rewards/rejected": -3.8165335655212402, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 1.598173515981735e-07, |
| "logits/chosen": -2.2415523529052734, |
| "logits/rejected": -1.9033533334732056, |
| "logps/chosen": -81.90745544433594, |
| "logps/rejected": -78.7509536743164, |
| "loss": 0.0511, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.247246503829956, |
| "rewards/margins": 5.400745391845703, |
| "rewards/rejected": -4.153499126434326, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.16, |
| "learning_rate": 1.643835616438356e-07, |
| "logits/chosen": -2.198545455932617, |
| "logits/rejected": -1.8948042392730713, |
| "logps/chosen": -85.43161010742188, |
| "logps/rejected": -76.58003997802734, |
| "loss": 0.0389, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3219624757766724, |
| "rewards/margins": 5.153651237487793, |
| "rewards/rejected": -3.8316891193389893, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 1.689497716894977e-07, |
| "logits/chosen": -2.288311004638672, |
| "logits/rejected": -1.879122018814087, |
| "logps/chosen": -88.61750793457031, |
| "logps/rejected": -73.41519927978516, |
| "loss": 0.0448, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.605739951133728, |
| "rewards/margins": 5.448731899261475, |
| "rewards/rejected": -3.8429923057556152, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.17, |
| "learning_rate": 1.735159817351598e-07, |
| "logits/chosen": -2.2476701736450195, |
| "logits/rejected": -1.837656021118164, |
| "logps/chosen": -88.90941619873047, |
| "logps/rejected": -76.27139282226562, |
| "loss": 0.027, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.609424352645874, |
| "rewards/margins": 6.268812656402588, |
| "rewards/rejected": -4.659388065338135, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 1.780821917808219e-07, |
| "logits/chosen": -2.256505012512207, |
| "logits/rejected": -1.9018818140029907, |
| "logps/chosen": -87.40512084960938, |
| "logps/rejected": -78.49671936035156, |
| "loss": 0.0365, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3038495779037476, |
| "rewards/margins": 5.88823938369751, |
| "rewards/rejected": -4.584390163421631, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.18, |
| "learning_rate": 1.82648401826484e-07, |
| "logits/chosen": -2.2472777366638184, |
| "logits/rejected": -1.9034316539764404, |
| "logps/chosen": -87.68411254882812, |
| "logps/rejected": -83.73296356201172, |
| "loss": 0.0284, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.4865331649780273, |
| "rewards/margins": 6.618298530578613, |
| "rewards/rejected": -5.131765842437744, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.18, |
| "eval_logits/chosen": -2.2571330070495605, |
| "eval_logits/rejected": -1.9162304401397705, |
| "eval_logps/chosen": -86.22880554199219, |
| "eval_logps/rejected": -77.06842803955078, |
| "eval_loss": 0.034718479961156845, |
| "eval_rewards/accuracies": 0.9832402467727661, |
| "eval_rewards/chosen": 1.300887942314148, |
| "eval_rewards/margins": 6.419642925262451, |
| "eval_rewards/rejected": -5.118754863739014, |
| "eval_runtime": 283.8557, |
| "eval_samples_per_second": 10.083, |
| "eval_steps_per_second": 0.631, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 1.872146118721461e-07, |
| "logits/chosen": -2.230461597442627, |
| "logits/rejected": -1.7934761047363281, |
| "logps/chosen": -95.64036560058594, |
| "logps/rejected": -79.05517578125, |
| "loss": 0.028, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 1.7179588079452515, |
| "rewards/margins": 6.512901306152344, |
| "rewards/rejected": -4.7949419021606445, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 1.917808219178082e-07, |
| "logits/chosen": -2.286062240600586, |
| "logits/rejected": -1.9140151739120483, |
| "logps/chosen": -85.09687042236328, |
| "logps/rejected": -79.01274871826172, |
| "loss": 0.0303, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2032204866409302, |
| "rewards/margins": 6.1523003578186035, |
| "rewards/rejected": -4.949079990386963, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 1.963470319634703e-07, |
| "logits/chosen": -2.296107053756714, |
| "logits/rejected": -2.0091071128845215, |
| "logps/chosen": -82.93006896972656, |
| "logps/rejected": -80.10784912109375, |
| "loss": 0.0268, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 1.1454616785049438, |
| "rewards/margins": 6.973064422607422, |
| "rewards/rejected": -5.827603340148926, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.2, |
| "learning_rate": 2.009132420091324e-07, |
| "logits/chosen": -2.2207303047180176, |
| "logits/rejected": -1.8763904571533203, |
| "logps/chosen": -82.83280181884766, |
| "logps/rejected": -79.92660522460938, |
| "loss": 0.0291, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.1346409320831299, |
| "rewards/margins": 7.1732892990112305, |
| "rewards/rejected": -6.03864860534668, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 2.054794520547945e-07, |
| "logits/chosen": -2.164816379547119, |
| "logits/rejected": -1.7991693019866943, |
| "logps/chosen": -91.07575988769531, |
| "logps/rejected": -82.17328643798828, |
| "loss": 0.0295, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.567754864692688, |
| "rewards/margins": 8.110410690307617, |
| "rewards/rejected": -6.542654991149902, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 2.100456621004566e-07, |
| "logits/chosen": -2.278972864151001, |
| "logits/rejected": -1.9182837009429932, |
| "logps/chosen": -91.42366790771484, |
| "logps/rejected": -88.95767974853516, |
| "loss": 0.0314, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.1288248300552368, |
| "rewards/margins": 7.684876441955566, |
| "rewards/rejected": -6.556051731109619, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.21, |
| "learning_rate": 2.146118721461187e-07, |
| "logits/chosen": -2.3190386295318604, |
| "logits/rejected": -1.961912751197815, |
| "logps/chosen": -86.6361312866211, |
| "logps/rejected": -81.09564971923828, |
| "loss": 0.0202, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5480232238769531, |
| "rewards/margins": 7.930706024169922, |
| "rewards/rejected": -6.382682800292969, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 2.191780821917808e-07, |
| "logits/chosen": -2.3331542015075684, |
| "logits/rejected": -1.9054124355316162, |
| "logps/chosen": -91.41481018066406, |
| "logps/rejected": -83.68817138671875, |
| "loss": 0.0218, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3129695653915405, |
| "rewards/margins": 8.34068775177002, |
| "rewards/rejected": -7.027717590332031, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.22, |
| "learning_rate": 2.237442922374429e-07, |
| "logits/chosen": -2.2767910957336426, |
| "logits/rejected": -1.91777765750885, |
| "logps/chosen": -90.29415130615234, |
| "logps/rejected": -84.94145965576172, |
| "loss": 0.017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.661865234375, |
| "rewards/margins": 8.428221702575684, |
| "rewards/rejected": -6.766357421875, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 2.28310502283105e-07, |
| "logits/chosen": -2.2446134090423584, |
| "logits/rejected": -1.8638511896133423, |
| "logps/chosen": -88.12340545654297, |
| "logps/rejected": -83.9640884399414, |
| "loss": 0.0164, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3303194046020508, |
| "rewards/margins": 8.892141342163086, |
| "rewards/rejected": -7.561821937561035, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.23, |
| "eval_logits/chosen": -2.269374132156372, |
| "eval_logits/rejected": -1.9262473583221436, |
| "eval_logps/chosen": -86.5768814086914, |
| "eval_logps/rejected": -82.61319732666016, |
| "eval_loss": 0.018967095762491226, |
| "eval_rewards/accuracies": 0.9860334992408752, |
| "eval_rewards/chosen": 1.126853346824646, |
| "eval_rewards/margins": 9.017987251281738, |
| "eval_rewards/rejected": -7.891134262084961, |
| "eval_runtime": 180.5741, |
| "eval_samples_per_second": 15.849, |
| "eval_steps_per_second": 0.991, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.23, |
| "learning_rate": 2.328767123287671e-07, |
| "logits/chosen": -2.2802085876464844, |
| "logits/rejected": -1.9185457229614258, |
| "logps/chosen": -81.77207946777344, |
| "logps/rejected": -80.97681427001953, |
| "loss": 0.0197, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.417055368423462, |
| "rewards/margins": 9.611557960510254, |
| "rewards/rejected": -8.194501876831055, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 2.374429223744292e-07, |
| "logits/chosen": -2.312295436859131, |
| "logits/rejected": -1.9153810739517212, |
| "logps/chosen": -91.66407775878906, |
| "logps/rejected": -87.04930877685547, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.3187003135681152, |
| "rewards/margins": 9.435572624206543, |
| "rewards/rejected": -8.11687183380127, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.24, |
| "learning_rate": 2.420091324200913e-07, |
| "logits/chosen": -2.347360134124756, |
| "logits/rejected": -1.9737800359725952, |
| "logps/chosen": -92.10830688476562, |
| "logps/rejected": -89.24776458740234, |
| "loss": 0.016, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5000791549682617, |
| "rewards/margins": 10.65346622467041, |
| "rewards/rejected": -9.153387069702148, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 2.465753424657534e-07, |
| "logits/chosen": -2.2161240577697754, |
| "logits/rejected": -1.819411039352417, |
| "logps/chosen": -89.9305648803711, |
| "logps/rejected": -87.90519714355469, |
| "loss": 0.0117, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.882406234741211, |
| "rewards/margins": 10.416455268859863, |
| "rewards/rejected": -8.534049034118652, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 2.511415525114155e-07, |
| "logits/chosen": -2.3072893619537354, |
| "logits/rejected": -1.928308129310608, |
| "logps/chosen": -88.01408386230469, |
| "logps/rejected": -83.06895446777344, |
| "loss": 0.0133, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1510813236236572, |
| "rewards/margins": 10.02571964263916, |
| "rewards/rejected": -8.87463665008545, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 2.557077625570776e-07, |
| "logits/chosen": -2.2718453407287598, |
| "logits/rejected": -1.8856366872787476, |
| "logps/chosen": -90.56663513183594, |
| "logps/rejected": -86.86662292480469, |
| "loss": 0.0137, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.4503662586212158, |
| "rewards/margins": 11.014634132385254, |
| "rewards/rejected": -9.5642671585083, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 2.602739726027397e-07, |
| "logits/chosen": -2.2816548347473145, |
| "logits/rejected": -1.8435026407241821, |
| "logps/chosen": -94.76255798339844, |
| "logps/rejected": -88.23798370361328, |
| "loss": 0.0171, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4174392223358154, |
| "rewards/margins": 10.936891555786133, |
| "rewards/rejected": -9.519450187683105, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.26, |
| "learning_rate": 2.648401826484018e-07, |
| "logits/chosen": -2.298529863357544, |
| "logits/rejected": -1.8752195835113525, |
| "logps/chosen": -88.09405517578125, |
| "logps/rejected": -88.1880874633789, |
| "loss": 0.0106, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.359735131263733, |
| "rewards/margins": 11.649232864379883, |
| "rewards/rejected": -10.289498329162598, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 2.694063926940639e-07, |
| "logits/chosen": -2.2168471813201904, |
| "logits/rejected": -1.908395528793335, |
| "logps/chosen": -89.10639953613281, |
| "logps/rejected": -90.54367065429688, |
| "loss": 0.0091, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3336710929870605, |
| "rewards/margins": 12.000402450561523, |
| "rewards/rejected": -10.666732788085938, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.27, |
| "learning_rate": 2.73972602739726e-07, |
| "logits/chosen": -2.3078534603118896, |
| "logits/rejected": -1.9278669357299805, |
| "logps/chosen": -85.4991683959961, |
| "logps/rejected": -89.22947692871094, |
| "loss": 0.0115, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5495243072509766, |
| "rewards/margins": 12.214988708496094, |
| "rewards/rejected": -10.665464401245117, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.27, |
| "eval_logits/chosen": -2.258920431137085, |
| "eval_logits/rejected": -1.9018107652664185, |
| "eval_logps/chosen": -86.94231414794922, |
| "eval_logps/rejected": -88.5583267211914, |
| "eval_loss": 0.01211754884570837, |
| "eval_rewards/accuracies": 0.9860334992408752, |
| "eval_rewards/chosen": 0.9441364407539368, |
| "eval_rewards/margins": 11.807838439941406, |
| "eval_rewards/rejected": -10.863702774047852, |
| "eval_runtime": 220.3515, |
| "eval_samples_per_second": 12.988, |
| "eval_steps_per_second": 0.812, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 2.785388127853881e-07, |
| "logits/chosen": -2.3034229278564453, |
| "logits/rejected": -1.9104959964752197, |
| "logps/chosen": -89.28177642822266, |
| "logps/rejected": -89.79473114013672, |
| "loss": 0.0112, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2887121438980103, |
| "rewards/margins": 11.897204399108887, |
| "rewards/rejected": -10.608492851257324, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.28, |
| "learning_rate": 2.831050228310502e-07, |
| "logits/chosen": -2.3329930305480957, |
| "logits/rejected": -1.9547516107559204, |
| "logps/chosen": -87.00670623779297, |
| "logps/rejected": -91.48796081542969, |
| "loss": 0.0116, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2360261678695679, |
| "rewards/margins": 12.600028991699219, |
| "rewards/rejected": -11.364001274108887, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 2.876712328767123e-07, |
| "logits/chosen": -2.298924446105957, |
| "logits/rejected": -1.964695930480957, |
| "logps/chosen": -91.54945373535156, |
| "logps/rejected": -96.8219985961914, |
| "loss": 0.0097, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5358260869979858, |
| "rewards/margins": 13.666036605834961, |
| "rewards/rejected": -12.130212783813477, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.29, |
| "learning_rate": 2.922374429223744e-07, |
| "logits/chosen": -2.3074567317962646, |
| "logits/rejected": -1.9056323766708374, |
| "logps/chosen": -89.87752532958984, |
| "logps/rejected": -90.3302230834961, |
| "loss": 0.0129, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.6936411261558533, |
| "rewards/margins": 11.324541091918945, |
| "rewards/rejected": -10.630899429321289, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 2.968036529680365e-07, |
| "logits/chosen": -2.310959815979004, |
| "logits/rejected": -1.8844079971313477, |
| "logps/chosen": -89.85198974609375, |
| "logps/rejected": -88.82755279541016, |
| "loss": 0.0041, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.92197585105896, |
| "rewards/margins": 13.281471252441406, |
| "rewards/rejected": -11.359495162963867, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.3, |
| "learning_rate": 2.998477929984779e-07, |
| "logits/chosen": -2.326324462890625, |
| "logits/rejected": -1.9201898574829102, |
| "logps/chosen": -86.74824523925781, |
| "logps/rejected": -93.16512298583984, |
| "loss": 0.0083, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1350452899932861, |
| "rewards/margins": 13.680140495300293, |
| "rewards/rejected": -12.545095443725586, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 2.993404363267377e-07, |
| "logits/chosen": -2.307976484298706, |
| "logits/rejected": -1.9049756526947021, |
| "logps/chosen": -91.18403625488281, |
| "logps/rejected": -93.76152801513672, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3413939476013184, |
| "rewards/margins": 14.000112533569336, |
| "rewards/rejected": -12.658717155456543, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 2.9883307965499743e-07, |
| "logits/chosen": -2.2899010181427, |
| "logits/rejected": -1.9124386310577393, |
| "logps/chosen": -91.78596496582031, |
| "logps/rejected": -97.47313690185547, |
| "loss": 0.0082, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.643972635269165, |
| "rewards/margins": 14.497393608093262, |
| "rewards/rejected": -12.853422164916992, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 2.983257229832572e-07, |
| "logits/chosen": -2.278542995452881, |
| "logits/rejected": -1.8497650623321533, |
| "logps/chosen": -89.2038803100586, |
| "logps/rejected": -89.34355926513672, |
| "loss": 0.0084, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.009284496307373, |
| "rewards/margins": 14.792158126831055, |
| "rewards/rejected": -12.782875061035156, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 2.9781836631151696e-07, |
| "logits/chosen": -2.255171060562134, |
| "logits/rejected": -1.9044491052627563, |
| "logps/chosen": -89.71915435791016, |
| "logps/rejected": -97.62041473388672, |
| "loss": 0.006, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.1625474691390991, |
| "rewards/margins": 13.750628471374512, |
| "rewards/rejected": -12.588080406188965, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.32, |
| "eval_logits/chosen": -2.2662999629974365, |
| "eval_logits/rejected": -1.9091564416885376, |
| "eval_logps/chosen": -86.36480712890625, |
| "eval_logps/rejected": -93.07726287841797, |
| "eval_loss": 0.00890163704752922, |
| "eval_rewards/accuracies": 0.9888268113136292, |
| "eval_rewards/chosen": 1.2328906059265137, |
| "eval_rewards/margins": 14.356060981750488, |
| "eval_rewards/rejected": -13.123170852661133, |
| "eval_runtime": 201.1096, |
| "eval_samples_per_second": 14.231, |
| "eval_steps_per_second": 0.89, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.32, |
| "learning_rate": 2.9731100963977676e-07, |
| "logits/chosen": -2.2573180198669434, |
| "logits/rejected": -1.8857762813568115, |
| "logps/chosen": -86.557861328125, |
| "logps/rejected": -94.78699493408203, |
| "loss": 0.0053, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.315459132194519, |
| "rewards/margins": 14.209829330444336, |
| "rewards/rejected": -12.894371032714844, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 2.968036529680365e-07, |
| "logits/chosen": -2.325437068939209, |
| "logits/rejected": -1.954026460647583, |
| "logps/chosen": -85.6775131225586, |
| "logps/rejected": -95.70713806152344, |
| "loss": 0.0078, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7716138362884521, |
| "rewards/margins": 15.646145820617676, |
| "rewards/rejected": -13.874531745910645, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.33, |
| "learning_rate": 2.962962962962963e-07, |
| "logits/chosen": -2.3093421459198, |
| "logits/rejected": -1.8865350484848022, |
| "logps/chosen": -86.63150787353516, |
| "logps/rejected": -93.17677307128906, |
| "loss": 0.0059, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0186963081359863, |
| "rewards/margins": 15.363825798034668, |
| "rewards/rejected": -13.345129013061523, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 2.9578893962455603e-07, |
| "logits/chosen": -2.2457454204559326, |
| "logits/rejected": -1.8727748394012451, |
| "logps/chosen": -84.77641296386719, |
| "logps/rejected": -90.2676010131836, |
| "loss": 0.0081, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4204827547073364, |
| "rewards/margins": 14.20788288116455, |
| "rewards/rejected": -12.787399291992188, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.34, |
| "learning_rate": 2.952815829528158e-07, |
| "logits/chosen": -2.308354616165161, |
| "logits/rejected": -1.9623510837554932, |
| "logps/chosen": -83.89095306396484, |
| "logps/rejected": -93.6553955078125, |
| "loss": 0.0102, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9429199695587158, |
| "rewards/margins": 14.841961860656738, |
| "rewards/rejected": -12.899042129516602, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 2.9477422628107556e-07, |
| "logits/chosen": -2.3661086559295654, |
| "logits/rejected": -1.9924736022949219, |
| "logps/chosen": -87.33646392822266, |
| "logps/rejected": -95.00645446777344, |
| "loss": 0.0145, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.3574576377868652, |
| "rewards/margins": 14.781530380249023, |
| "rewards/rejected": -12.424072265625, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.35, |
| "learning_rate": 2.9426686960933536e-07, |
| "logits/chosen": -2.2825839519500732, |
| "logits/rejected": -1.8536045551300049, |
| "logps/chosen": -93.97390747070312, |
| "logps/rejected": -94.27307891845703, |
| "loss": 0.008, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.272422194480896, |
| "rewards/margins": 14.426435470581055, |
| "rewards/rejected": -13.154012680053711, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 2.937595129375951e-07, |
| "logits/chosen": -2.228724956512451, |
| "logits/rejected": -1.8590246438980103, |
| "logps/chosen": -86.09639739990234, |
| "logps/rejected": -92.32320404052734, |
| "loss": 0.0058, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.6447618007659912, |
| "rewards/margins": 14.473971366882324, |
| "rewards/rejected": -12.82921028137207, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.36, |
| "learning_rate": 2.932521562658549e-07, |
| "logits/chosen": -2.298750400543213, |
| "logits/rejected": -1.9722936153411865, |
| "logps/chosen": -88.01515197753906, |
| "logps/rejected": -96.683349609375, |
| "loss": 0.0066, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.2021758556365967, |
| "rewards/margins": 15.400471687316895, |
| "rewards/rejected": -13.198295593261719, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 2.9274479959411463e-07, |
| "logits/chosen": -2.2984695434570312, |
| "logits/rejected": -1.9286425113677979, |
| "logps/chosen": -84.20467376708984, |
| "logps/rejected": -96.1876449584961, |
| "loss": 0.0068, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.15755558013916, |
| "rewards/margins": 15.103589057922363, |
| "rewards/rejected": -12.946032524108887, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.37, |
| "eval_logits/chosen": -2.2589728832244873, |
| "eval_logits/rejected": -1.9111113548278809, |
| "eval_logps/chosen": -85.02589416503906, |
| "eval_logps/rejected": -92.66677856445312, |
| "eval_loss": 0.00817781314253807, |
| "eval_rewards/accuracies": 0.9888268113136292, |
| "eval_rewards/chosen": 1.902347207069397, |
| "eval_rewards/margins": 14.820280075073242, |
| "eval_rewards/rejected": -12.917930603027344, |
| "eval_runtime": 235.1892, |
| "eval_samples_per_second": 12.169, |
| "eval_steps_per_second": 0.761, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 2.922374429223744e-07, |
| "logits/chosen": -2.246121883392334, |
| "logits/rejected": -1.878603219985962, |
| "logps/chosen": -84.80223083496094, |
| "logps/rejected": -96.83042907714844, |
| "loss": 0.0154, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4696574211120605, |
| "rewards/margins": 15.41651439666748, |
| "rewards/rejected": -12.946856498718262, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.37, |
| "learning_rate": 2.9173008625063416e-07, |
| "logits/chosen": -2.2706353664398193, |
| "logits/rejected": -1.885000467300415, |
| "logps/chosen": -88.46263122558594, |
| "logps/rejected": -95.60710144042969, |
| "loss": 0.0061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.5691077709198, |
| "rewards/margins": 16.42782211303711, |
| "rewards/rejected": -13.858716011047363, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 2.9122272957889396e-07, |
| "logits/chosen": -2.2218751907348633, |
| "logits/rejected": -1.8037551641464233, |
| "logps/chosen": -89.38809204101562, |
| "logps/rejected": -99.0447006225586, |
| "loss": 0.0107, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.3626868724823, |
| "rewards/margins": 15.406193733215332, |
| "rewards/rejected": -13.04350757598877, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 2.907153729071537e-07, |
| "logits/chosen": -2.2497596740722656, |
| "logits/rejected": -1.7971522808074951, |
| "logps/chosen": -91.6142807006836, |
| "logps/rejected": -95.75736236572266, |
| "loss": 0.0077, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9224580526351929, |
| "rewards/margins": 16.020370483398438, |
| "rewards/rejected": -14.09791088104248, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 2.902080162354135e-07, |
| "logits/chosen": -2.2165656089782715, |
| "logits/rejected": -1.902951955795288, |
| "logps/chosen": -79.86293029785156, |
| "logps/rejected": -97.59088134765625, |
| "loss": 0.0073, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5969226360321045, |
| "rewards/margins": 15.788568496704102, |
| "rewards/rejected": -14.191644668579102, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.39, |
| "learning_rate": 2.8970065956367323e-07, |
| "logits/chosen": -2.272202968597412, |
| "logits/rejected": -1.8873332738876343, |
| "logps/chosen": -87.25418853759766, |
| "logps/rejected": -99.46211242675781, |
| "loss": 0.0108, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7722752094268799, |
| "rewards/margins": 16.73154640197754, |
| "rewards/rejected": -14.959269523620605, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 2.89193302891933e-07, |
| "logits/chosen": -2.255979061126709, |
| "logits/rejected": -1.9493907690048218, |
| "logps/chosen": -86.21080017089844, |
| "logps/rejected": -106.2105941772461, |
| "loss": 0.008, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.9701576232910156, |
| "rewards/margins": 18.300521850585938, |
| "rewards/rejected": -17.330366134643555, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.4, |
| "learning_rate": 2.8868594622019276e-07, |
| "logits/chosen": -2.2402877807617188, |
| "logits/rejected": -1.801593542098999, |
| "logps/chosen": -91.86135864257812, |
| "logps/rejected": -99.11498260498047, |
| "loss": 0.0049, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.0486648082733154, |
| "rewards/margins": 17.628559112548828, |
| "rewards/rejected": -15.579893112182617, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 2.8817858954845256e-07, |
| "logits/chosen": -2.2116286754608154, |
| "logits/rejected": -1.8907750844955444, |
| "logps/chosen": -86.34685516357422, |
| "logps/rejected": -105.34354400634766, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.269397735595703, |
| "rewards/margins": 18.360477447509766, |
| "rewards/rejected": -16.091083526611328, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.41, |
| "learning_rate": 2.876712328767123e-07, |
| "logits/chosen": -2.2881391048431396, |
| "logits/rejected": -1.9034688472747803, |
| "logps/chosen": -85.5262222290039, |
| "logps/rejected": -103.30674743652344, |
| "loss": 0.0047, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0433995723724365, |
| "rewards/margins": 19.068531036376953, |
| "rewards/rejected": -17.025129318237305, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.41, |
| "eval_logits/chosen": -2.2466113567352295, |
| "eval_logits/rejected": -1.893018364906311, |
| "eval_logps/chosen": -85.62364959716797, |
| "eval_logps/rejected": -100.12027740478516, |
| "eval_loss": 0.007255914621055126, |
| "eval_rewards/accuracies": 0.9888268113136292, |
| "eval_rewards/chosen": 1.6034660339355469, |
| "eval_rewards/margins": 18.248144149780273, |
| "eval_rewards/rejected": -16.644678115844727, |
| "eval_runtime": 217.5149, |
| "eval_samples_per_second": 13.158, |
| "eval_steps_per_second": 0.823, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 2.871638762049721e-07, |
| "logits/chosen": -2.2870774269104004, |
| "logits/rejected": -1.9350963830947876, |
| "logps/chosen": -92.18787384033203, |
| "logps/rejected": -107.0401611328125, |
| "loss": 0.0056, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6550416946411133, |
| "rewards/margins": 18.991727828979492, |
| "rewards/rejected": -17.336687088012695, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 2.8665651953323183e-07, |
| "logits/chosen": -2.281367063522339, |
| "logits/rejected": -1.8545904159545898, |
| "logps/chosen": -85.82614135742188, |
| "logps/rejected": -100.07356262207031, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7310631275177, |
| "rewards/margins": 19.34614372253418, |
| "rewards/rejected": -16.615079879760742, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.42, |
| "learning_rate": 2.861491628614916e-07, |
| "logits/chosen": -2.313906192779541, |
| "logits/rejected": -1.910033941268921, |
| "logps/chosen": -85.05974578857422, |
| "logps/rejected": -101.42959594726562, |
| "loss": 0.0071, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.475045919418335, |
| "rewards/margins": 17.93846893310547, |
| "rewards/rejected": -16.463422775268555, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 2.8564180618975136e-07, |
| "logits/chosen": -2.234340190887451, |
| "logits/rejected": -1.8709052801132202, |
| "logps/chosen": -90.11567687988281, |
| "logps/rejected": -107.35555267333984, |
| "loss": 0.005, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.179823398590088, |
| "rewards/margins": 19.71477699279785, |
| "rewards/rejected": -17.53495216369629, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.43, |
| "learning_rate": 2.8513444951801116e-07, |
| "logits/chosen": -2.2462267875671387, |
| "logits/rejected": -1.8993927240371704, |
| "logps/chosen": -86.99429321289062, |
| "logps/rejected": -108.23291015625, |
| "loss": 0.0068, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4878250360488892, |
| "rewards/margins": 20.112104415893555, |
| "rewards/rejected": -18.624279022216797, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 2.846270928462709e-07, |
| "logits/chosen": -2.196100950241089, |
| "logits/rejected": -1.8292429447174072, |
| "logps/chosen": -89.53839874267578, |
| "logps/rejected": -103.94474029541016, |
| "loss": 0.0019, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.670432686805725, |
| "rewards/margins": 18.502605438232422, |
| "rewards/rejected": -16.832172393798828, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 2.841197361745307e-07, |
| "logits/chosen": -2.222852945327759, |
| "logits/rejected": -1.827455759048462, |
| "logps/chosen": -87.63113403320312, |
| "logps/rejected": -101.9697265625, |
| "loss": 0.0086, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.7835509777069092, |
| "rewards/margins": 19.11504364013672, |
| "rewards/rejected": -17.331493377685547, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 2.8361237950279043e-07, |
| "logits/chosen": -2.178133726119995, |
| "logits/rejected": -1.8010485172271729, |
| "logps/chosen": -87.01272583007812, |
| "logps/rejected": -104.30177307128906, |
| "loss": 0.0123, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2377567291259766, |
| "rewards/margins": 18.971233367919922, |
| "rewards/rejected": -16.733478546142578, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.45, |
| "learning_rate": 2.831050228310502e-07, |
| "logits/chosen": -2.2163853645324707, |
| "logits/rejected": -1.8224786520004272, |
| "logps/chosen": -85.6602554321289, |
| "logps/rejected": -103.28083801269531, |
| "loss": 0.006, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.684990882873535, |
| "rewards/margins": 18.973764419555664, |
| "rewards/rejected": -16.288774490356445, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 2.8259766615930996e-07, |
| "logits/chosen": -2.2510056495666504, |
| "logits/rejected": -1.7727775573730469, |
| "logps/chosen": -87.45413970947266, |
| "logps/rejected": -97.86138916015625, |
| "loss": 0.0053, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.894382953643799, |
| "rewards/margins": 19.050537109375, |
| "rewards/rejected": -16.15615463256836, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.46, |
| "eval_logits/chosen": -2.2001523971557617, |
| "eval_logits/rejected": -1.8469951152801514, |
| "eval_logps/chosen": -84.38460540771484, |
| "eval_logps/rejected": -99.33098602294922, |
| "eval_loss": 0.0070889778435230255, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.222993850708008, |
| "eval_rewards/margins": 18.473024368286133, |
| "eval_rewards/rejected": -16.250030517578125, |
| "eval_runtime": 165.1867, |
| "eval_samples_per_second": 17.326, |
| "eval_steps_per_second": 1.084, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.46, |
| "learning_rate": 2.8209030948756976e-07, |
| "logits/chosen": -2.2496349811553955, |
| "logits/rejected": -1.8998152017593384, |
| "logps/chosen": -84.53297424316406, |
| "logps/rejected": -103.37425231933594, |
| "loss": 0.005, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.839362621307373, |
| "rewards/margins": 18.76103401184082, |
| "rewards/rejected": -15.921670913696289, |
| "step": 1010 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 2.815829528158295e-07, |
| "logits/chosen": -2.176231861114502, |
| "logits/rejected": -1.815509557723999, |
| "logps/chosen": -83.98689270019531, |
| "logps/rejected": -102.61014556884766, |
| "loss": 0.0039, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5853066444396973, |
| "rewards/margins": 19.177194595336914, |
| "rewards/rejected": -16.591888427734375, |
| "step": 1020 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 2.810755961440893e-07, |
| "logits/chosen": -2.2483859062194824, |
| "logits/rejected": -1.837993860244751, |
| "logps/chosen": -85.09949493408203, |
| "logps/rejected": -105.0072021484375, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.028395175933838, |
| "rewards/margins": 19.074552536010742, |
| "rewards/rejected": -17.046157836914062, |
| "step": 1030 |
| }, |
| { |
| "epoch": 0.47, |
| "learning_rate": 2.8056823947234903e-07, |
| "logits/chosen": -2.1420371532440186, |
| "logits/rejected": -1.7370542287826538, |
| "logps/chosen": -87.30947875976562, |
| "logps/rejected": -97.69217681884766, |
| "loss": 0.0078, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.3234381675720215, |
| "rewards/margins": 18.487682342529297, |
| "rewards/rejected": -15.16424560546875, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 2.800608828006088e-07, |
| "logits/chosen": -2.2500481605529785, |
| "logits/rejected": -1.8463836908340454, |
| "logps/chosen": -90.0582275390625, |
| "logps/rejected": -103.7062759399414, |
| "loss": 0.0053, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3003077507019043, |
| "rewards/margins": 19.799549102783203, |
| "rewards/rejected": -17.49924087524414, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.48, |
| "learning_rate": 2.7955352612886856e-07, |
| "logits/chosen": -2.208354949951172, |
| "logits/rejected": -1.8978168964385986, |
| "logps/chosen": -77.57411193847656, |
| "logps/rejected": -103.87590026855469, |
| "loss": 0.007, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.4448658227920532, |
| "rewards/margins": 20.587467193603516, |
| "rewards/rejected": -19.142602920532227, |
| "step": 1060 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 2.7904616945712836e-07, |
| "logits/chosen": -2.1659793853759766, |
| "logits/rejected": -1.7798793315887451, |
| "logps/chosen": -83.9134521484375, |
| "logps/rejected": -107.64317321777344, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1447739601135254, |
| "rewards/margins": 21.296142578125, |
| "rewards/rejected": -19.151369094848633, |
| "step": 1070 |
| }, |
| { |
| "epoch": 0.49, |
| "learning_rate": 2.785388127853881e-07, |
| "logits/chosen": -2.2320501804351807, |
| "logits/rejected": -1.8187439441680908, |
| "logps/chosen": -88.16758728027344, |
| "logps/rejected": -105.4022445678711, |
| "loss": 0.0082, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6898410320281982, |
| "rewards/margins": 21.423625946044922, |
| "rewards/rejected": -18.733787536621094, |
| "step": 1080 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.780314561136479e-07, |
| "logits/chosen": -2.150700092315674, |
| "logits/rejected": -1.8009631633758545, |
| "logps/chosen": -88.3268051147461, |
| "logps/rejected": -107.76560974121094, |
| "loss": 0.0057, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.4143996238708496, |
| "rewards/margins": 20.30838394165039, |
| "rewards/rejected": -18.893983840942383, |
| "step": 1090 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.7752409944190763e-07, |
| "logits/chosen": -2.2560830116271973, |
| "logits/rejected": -1.8415091037750244, |
| "logps/chosen": -88.06275939941406, |
| "logps/rejected": -104.66233825683594, |
| "loss": 0.0082, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9098188877105713, |
| "rewards/margins": 20.39904022216797, |
| "rewards/rejected": -18.489221572875977, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.5, |
| "eval_logits/chosen": -2.208942413330078, |
| "eval_logits/rejected": -1.8563501834869385, |
| "eval_logps/chosen": -84.56623840332031, |
| "eval_logps/rejected": -100.85079956054688, |
| "eval_loss": 0.007256262004375458, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.132176637649536, |
| "eval_rewards/margins": 19.14211654663086, |
| "eval_rewards/rejected": -17.00994110107422, |
| "eval_runtime": 179.5596, |
| "eval_samples_per_second": 15.939, |
| "eval_steps_per_second": 0.997, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.770167427701674e-07, |
| "logits/chosen": -2.148176431655884, |
| "logits/rejected": -1.811261534690857, |
| "logps/chosen": -90.28803253173828, |
| "logps/rejected": -105.98939514160156, |
| "loss": 0.0037, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.97528874874115, |
| "rewards/margins": 18.622446060180664, |
| "rewards/rejected": -16.647159576416016, |
| "step": 1110 |
| }, |
| { |
| "epoch": 0.51, |
| "learning_rate": 2.7650938609842716e-07, |
| "logits/chosen": -2.226710796356201, |
| "logits/rejected": -1.8027299642562866, |
| "logps/chosen": -88.32550048828125, |
| "logps/rejected": -98.92311096191406, |
| "loss": 0.0021, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.893244504928589, |
| "rewards/margins": 20.122953414916992, |
| "rewards/rejected": -17.22970962524414, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.7600202942668696e-07, |
| "logits/chosen": -2.259479284286499, |
| "logits/rejected": -1.845926284790039, |
| "logps/chosen": -90.93453216552734, |
| "logps/rejected": -101.05252075195312, |
| "loss": 0.0021, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7073707580566406, |
| "rewards/margins": 19.56781578063965, |
| "rewards/rejected": -16.860445022583008, |
| "step": 1130 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.754946727549467e-07, |
| "logits/chosen": -2.253920793533325, |
| "logits/rejected": -1.9071108102798462, |
| "logps/chosen": -89.0732650756836, |
| "logps/rejected": -106.84078216552734, |
| "loss": 0.0072, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.451443672180176, |
| "rewards/margins": 19.57352638244629, |
| "rewards/rejected": -17.122081756591797, |
| "step": 1140 |
| }, |
| { |
| "epoch": 0.52, |
| "learning_rate": 2.749873160832065e-07, |
| "logits/chosen": -2.217794418334961, |
| "logits/rejected": -1.8264172077178955, |
| "logps/chosen": -84.98257446289062, |
| "logps/rejected": -103.6314926147461, |
| "loss": 0.002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.75927734375, |
| "rewards/margins": 20.883607864379883, |
| "rewards/rejected": -18.12432861328125, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.7447995941146623e-07, |
| "logits/chosen": -2.23350191116333, |
| "logits/rejected": -1.8721933364868164, |
| "logps/chosen": -88.2120361328125, |
| "logps/rejected": -107.16935729980469, |
| "loss": 0.0196, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 2.1735239028930664, |
| "rewards/margins": 19.927579879760742, |
| "rewards/rejected": -17.75405502319336, |
| "step": 1160 |
| }, |
| { |
| "epoch": 0.53, |
| "learning_rate": 2.73972602739726e-07, |
| "logits/chosen": -2.191521167755127, |
| "logits/rejected": -1.8549985885620117, |
| "logps/chosen": -88.76947784423828, |
| "logps/rejected": -109.21342468261719, |
| "loss": 0.0061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.1197288036346436, |
| "rewards/margins": 21.133193969726562, |
| "rewards/rejected": -19.013463973999023, |
| "step": 1170 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.7346524606798576e-07, |
| "logits/chosen": -2.223162889480591, |
| "logits/rejected": -1.8733783960342407, |
| "logps/chosen": -88.88787841796875, |
| "logps/rejected": -108.9013671875, |
| "loss": 0.0062, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.40854811668396, |
| "rewards/margins": 22.095478057861328, |
| "rewards/rejected": -19.686931610107422, |
| "step": 1180 |
| }, |
| { |
| "epoch": 0.54, |
| "learning_rate": 2.7295788939624556e-07, |
| "logits/chosen": -2.197874069213867, |
| "logits/rejected": -1.7851364612579346, |
| "logps/chosen": -85.47915649414062, |
| "logps/rejected": -101.67639923095703, |
| "loss": 0.0049, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8164796829223633, |
| "rewards/margins": 20.382543563842773, |
| "rewards/rejected": -17.566064834594727, |
| "step": 1190 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.724505327245053e-07, |
| "logits/chosen": -2.2712106704711914, |
| "logits/rejected": -1.9540023803710938, |
| "logps/chosen": -88.18670654296875, |
| "logps/rejected": -107.9418716430664, |
| "loss": 0.0243, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.7379881143569946, |
| "rewards/margins": 19.269012451171875, |
| "rewards/rejected": -17.531024932861328, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.55, |
| "eval_logits/chosen": -2.1799449920654297, |
| "eval_logits/rejected": -1.832864761352539, |
| "eval_logps/chosen": -83.93651580810547, |
| "eval_logps/rejected": -97.79573822021484, |
| "eval_loss": 0.007186249829828739, |
| "eval_rewards/accuracies": 0.9888268113136292, |
| "eval_rewards/chosen": 2.4470374584198, |
| "eval_rewards/margins": 17.92945098876953, |
| "eval_rewards/rejected": -15.482412338256836, |
| "eval_runtime": 229.0536, |
| "eval_samples_per_second": 12.495, |
| "eval_steps_per_second": 0.781, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.55, |
| "learning_rate": 2.719431760527651e-07, |
| "logits/chosen": -2.1936728954315186, |
| "logits/rejected": -1.8306325674057007, |
| "logps/chosen": -84.01075744628906, |
| "logps/rejected": -96.85215759277344, |
| "loss": 0.0029, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3379154205322266, |
| "rewards/margins": 16.724027633666992, |
| "rewards/rejected": -14.38611125946045, |
| "step": 1210 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.7143581938102483e-07, |
| "logits/chosen": -2.2218477725982666, |
| "logits/rejected": -1.848719835281372, |
| "logps/chosen": -81.31999969482422, |
| "logps/rejected": -99.01549530029297, |
| "loss": 0.0051, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9159164428710938, |
| "rewards/margins": 18.738767623901367, |
| "rewards/rejected": -15.822851181030273, |
| "step": 1220 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.709284627092846e-07, |
| "logits/chosen": -2.1815645694732666, |
| "logits/rejected": -1.8792476654052734, |
| "logps/chosen": -81.18501281738281, |
| "logps/rejected": -99.95542907714844, |
| "loss": 0.0038, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6159422397613525, |
| "rewards/margins": 18.44463348388672, |
| "rewards/rejected": -15.828694343566895, |
| "step": 1230 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.7042110603754436e-07, |
| "logits/chosen": -2.199174642562866, |
| "logits/rejected": -1.7883743047714233, |
| "logps/chosen": -86.32429504394531, |
| "logps/rejected": -99.67549133300781, |
| "loss": 0.0066, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.1772170066833496, |
| "rewards/margins": 19.528255462646484, |
| "rewards/rejected": -16.35103988647461, |
| "step": 1240 |
| }, |
| { |
| "epoch": 0.57, |
| "learning_rate": 2.6991374936580416e-07, |
| "logits/chosen": -2.1487929821014404, |
| "logits/rejected": -1.789044737815857, |
| "logps/chosen": -85.71726989746094, |
| "logps/rejected": -104.7240219116211, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.469191312789917, |
| "rewards/margins": 19.757835388183594, |
| "rewards/rejected": -17.28864288330078, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.694063926940639e-07, |
| "logits/chosen": -2.1753883361816406, |
| "logits/rejected": -1.798710823059082, |
| "logps/chosen": -87.18423461914062, |
| "logps/rejected": -101.5936279296875, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.384445905685425, |
| "rewards/margins": 19.821514129638672, |
| "rewards/rejected": -17.43706512451172, |
| "step": 1260 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.688990360223237e-07, |
| "logits/chosen": -2.170189619064331, |
| "logits/rejected": -1.8727480173110962, |
| "logps/chosen": -84.32581329345703, |
| "logps/rejected": -109.3947982788086, |
| "loss": 0.008, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.941753625869751, |
| "rewards/margins": 20.492143630981445, |
| "rewards/rejected": -18.55038833618164, |
| "step": 1270 |
| }, |
| { |
| "epoch": 0.58, |
| "learning_rate": 2.6839167935058343e-07, |
| "logits/chosen": -2.334543466567993, |
| "logits/rejected": -1.9388000965118408, |
| "logps/chosen": -83.55552673339844, |
| "logps/rejected": -104.6715316772461, |
| "loss": 0.0091, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.3667588233947754, |
| "rewards/margins": 21.2448787689209, |
| "rewards/rejected": -18.87812042236328, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.678843226788432e-07, |
| "logits/chosen": -2.1919655799865723, |
| "logits/rejected": -1.857973337173462, |
| "logps/chosen": -89.53315734863281, |
| "logps/rejected": -108.6443099975586, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.031522274017334, |
| "rewards/margins": 21.71250343322754, |
| "rewards/rejected": -19.680980682373047, |
| "step": 1290 |
| }, |
| { |
| "epoch": 0.59, |
| "learning_rate": 2.6737696600710296e-07, |
| "logits/chosen": -2.2731220722198486, |
| "logits/rejected": -1.894622802734375, |
| "logps/chosen": -80.3070297241211, |
| "logps/rejected": -106.09849548339844, |
| "loss": 0.0053, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.022993564605713, |
| "rewards/margins": 22.860183715820312, |
| "rewards/rejected": -19.837190628051758, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.59, |
| "eval_logits/chosen": -2.197847366333008, |
| "eval_logits/rejected": -1.8479573726654053, |
| "eval_logps/chosen": -85.01256561279297, |
| "eval_logps/rejected": -106.13009643554688, |
| "eval_loss": 0.007005217019468546, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 1.9090105295181274, |
| "eval_rewards/margins": 21.558591842651367, |
| "eval_rewards/rejected": -19.64958381652832, |
| "eval_runtime": 244.0952, |
| "eval_samples_per_second": 11.725, |
| "eval_steps_per_second": 0.733, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.6686960933536276e-07, |
| "logits/chosen": -2.257427215576172, |
| "logits/rejected": -1.9010261297225952, |
| "logps/chosen": -85.78707885742188, |
| "logps/rejected": -110.6099853515625, |
| "loss": 0.0068, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4400665760040283, |
| "rewards/margins": 22.056560516357422, |
| "rewards/rejected": -19.616491317749023, |
| "step": 1310 |
| }, |
| { |
| "epoch": 0.6, |
| "learning_rate": 2.663622526636225e-07, |
| "logits/chosen": -2.1522250175476074, |
| "logits/rejected": -1.8013957738876343, |
| "logps/chosen": -85.6783447265625, |
| "logps/rejected": -111.1842269897461, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.6612367630004883, |
| "rewards/margins": 22.052217483520508, |
| "rewards/rejected": -20.390979766845703, |
| "step": 1320 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 2.658548959918823e-07, |
| "logits/chosen": -2.207953691482544, |
| "logits/rejected": -1.8331537246704102, |
| "logps/chosen": -89.45965576171875, |
| "logps/rejected": -109.67543029785156, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7361319065093994, |
| "rewards/margins": 22.78788185119629, |
| "rewards/rejected": -20.051748275756836, |
| "step": 1330 |
| }, |
| { |
| "epoch": 0.61, |
| "learning_rate": 2.6534753932014203e-07, |
| "logits/chosen": -2.213942050933838, |
| "logits/rejected": -1.8521515130996704, |
| "logps/chosen": -85.9124526977539, |
| "logps/rejected": -104.89479064941406, |
| "loss": 0.0096, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.157468557357788, |
| "rewards/margins": 21.968364715576172, |
| "rewards/rejected": -19.810897827148438, |
| "step": 1340 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 2.648401826484018e-07, |
| "logits/chosen": -2.2531790733337402, |
| "logits/rejected": -1.883050560951233, |
| "logps/chosen": -83.91947937011719, |
| "logps/rejected": -117.13653564453125, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.926668405532837, |
| "rewards/margins": 24.423267364501953, |
| "rewards/rejected": -22.496599197387695, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.62, |
| "learning_rate": 2.6433282597666156e-07, |
| "logits/chosen": -2.1308257579803467, |
| "logits/rejected": -1.7693058252334595, |
| "logps/chosen": -89.39836120605469, |
| "logps/rejected": -115.6990737915039, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7298908233642578, |
| "rewards/margins": 23.361858367919922, |
| "rewards/rejected": -21.631967544555664, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 2.6382546930492135e-07, |
| "logits/chosen": -2.2151737213134766, |
| "logits/rejected": -1.8560287952423096, |
| "logps/chosen": -87.30859375, |
| "logps/rejected": -111.68087005615234, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5496299266815186, |
| "rewards/margins": 22.752437591552734, |
| "rewards/rejected": -21.202808380126953, |
| "step": 1370 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 2.633181126331811e-07, |
| "logits/chosen": -2.2534372806549072, |
| "logits/rejected": -1.9297653436660767, |
| "logps/chosen": -83.9007568359375, |
| "logps/rejected": -111.0853500366211, |
| "loss": 0.0061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8405357599258423, |
| "rewards/margins": 22.654870986938477, |
| "rewards/rejected": -20.814334869384766, |
| "step": 1380 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 2.628107559614409e-07, |
| "logits/chosen": -2.269160747528076, |
| "logits/rejected": -1.8955166339874268, |
| "logps/chosen": -85.63101196289062, |
| "logps/rejected": -112.61944580078125, |
| "loss": 0.001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2250313758850098, |
| "rewards/margins": 22.996959686279297, |
| "rewards/rejected": -20.771930694580078, |
| "step": 1390 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 2.6230339928970063e-07, |
| "logits/chosen": -2.2950210571289062, |
| "logits/rejected": -1.9462471008300781, |
| "logps/chosen": -88.02154541015625, |
| "logps/rejected": -110.0986557006836, |
| "loss": 0.0051, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7983818054199219, |
| "rewards/margins": 22.378314971923828, |
| "rewards/rejected": -20.57993507385254, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.64, |
| "eval_logits/chosen": -2.209514856338501, |
| "eval_logits/rejected": -1.8633878231048584, |
| "eval_logps/chosen": -85.35096740722656, |
| "eval_logps/rejected": -108.35541534423828, |
| "eval_loss": 0.007250170689076185, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.7398098707199097, |
| "eval_rewards/margins": 22.50205421447754, |
| "eval_rewards/rejected": -20.762245178222656, |
| "eval_runtime": 186.4436, |
| "eval_samples_per_second": 15.35, |
| "eval_steps_per_second": 0.96, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.64, |
| "learning_rate": 2.617960426179604e-07, |
| "logits/chosen": -2.224945306777954, |
| "logits/rejected": -1.8185851573944092, |
| "logps/chosen": -91.3012466430664, |
| "logps/rejected": -112.8142318725586, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3502469062805176, |
| "rewards/margins": 24.02829933166504, |
| "rewards/rejected": -21.678050994873047, |
| "step": 1410 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 2.6128868594622016e-07, |
| "logits/chosen": -2.163133144378662, |
| "logits/rejected": -1.816178560256958, |
| "logps/chosen": -85.406494140625, |
| "logps/rejected": -105.5617904663086, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.699741005897522, |
| "rewards/margins": 21.171184539794922, |
| "rewards/rejected": -19.47144317626953, |
| "step": 1420 |
| }, |
| { |
| "epoch": 0.65, |
| "learning_rate": 2.6078132927447995e-07, |
| "logits/chosen": -2.240720748901367, |
| "logits/rejected": -1.8999313116073608, |
| "logps/chosen": -86.69328308105469, |
| "logps/rejected": -114.80928802490234, |
| "loss": 0.0016, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3369460105895996, |
| "rewards/margins": 24.97016143798828, |
| "rewards/rejected": -22.633216857910156, |
| "step": 1430 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 2.602739726027397e-07, |
| "logits/chosen": -2.2251267433166504, |
| "logits/rejected": -1.8492538928985596, |
| "logps/chosen": -85.41273498535156, |
| "logps/rejected": -113.72379302978516, |
| "loss": 0.0027, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7388935089111328, |
| "rewards/margins": 24.013940811157227, |
| "rewards/rejected": -22.27504539489746, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.66, |
| "learning_rate": 2.597666159309995e-07, |
| "logits/chosen": -2.196199655532837, |
| "logits/rejected": -1.8432044982910156, |
| "logps/chosen": -84.84178924560547, |
| "logps/rejected": -113.02108001708984, |
| "loss": 0.0143, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.910146713256836, |
| "rewards/margins": 22.436695098876953, |
| "rewards/rejected": -20.52655029296875, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 2.5925925925925923e-07, |
| "logits/chosen": -2.2136662006378174, |
| "logits/rejected": -1.8691514730453491, |
| "logps/chosen": -87.48178100585938, |
| "logps/rejected": -113.52275085449219, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.627357840538025, |
| "rewards/margins": 21.706161499023438, |
| "rewards/rejected": -20.07880210876465, |
| "step": 1460 |
| }, |
| { |
| "epoch": 0.67, |
| "learning_rate": 2.58751902587519e-07, |
| "logits/chosen": -2.1794159412384033, |
| "logits/rejected": -1.8281217813491821, |
| "logps/chosen": -91.26468658447266, |
| "logps/rejected": -111.00882720947266, |
| "loss": 0.0065, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3432388305664062, |
| "rewards/margins": 22.417926788330078, |
| "rewards/rejected": -20.074687957763672, |
| "step": 1470 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 2.5824454591577876e-07, |
| "logits/chosen": -2.2314205169677734, |
| "logits/rejected": -1.8225589990615845, |
| "logps/chosen": -88.60221862792969, |
| "logps/rejected": -109.89363098144531, |
| "loss": 0.002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.573441982269287, |
| "rewards/margins": 24.366378784179688, |
| "rewards/rejected": -21.79293441772461, |
| "step": 1480 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 2.5773718924403855e-07, |
| "logits/chosen": -2.1291966438293457, |
| "logits/rejected": -1.784570336341858, |
| "logps/chosen": -87.0352554321289, |
| "logps/rejected": -114.33548736572266, |
| "loss": 0.0029, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9369707107543945, |
| "rewards/margins": 24.22989845275879, |
| "rewards/rejected": -22.292926788330078, |
| "step": 1490 |
| }, |
| { |
| "epoch": 0.68, |
| "learning_rate": 2.572298325722983e-07, |
| "logits/chosen": -2.148015260696411, |
| "logits/rejected": -1.8161808252334595, |
| "logps/chosen": -87.71135711669922, |
| "logps/rejected": -112.76509094238281, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6630229949951172, |
| "rewards/margins": 23.89804458618164, |
| "rewards/rejected": -22.235023498535156, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.68, |
| "eval_logits/chosen": -2.1937787532806396, |
| "eval_logits/rejected": -1.8467158079147339, |
| "eval_logps/chosen": -85.40876007080078, |
| "eval_logps/rejected": -109.89164733886719, |
| "eval_loss": 0.007178114727139473, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.7109133005142212, |
| "eval_rewards/margins": 23.241273880004883, |
| "eval_rewards/rejected": -21.530363082885742, |
| "eval_runtime": 183.2701, |
| "eval_samples_per_second": 15.616, |
| "eval_steps_per_second": 0.977, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 2.567224759005581e-07, |
| "logits/chosen": -2.22023344039917, |
| "logits/rejected": -1.8317985534667969, |
| "logps/chosen": -81.38858795166016, |
| "logps/rejected": -102.5715560913086, |
| "loss": 0.0068, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5601766109466553, |
| "rewards/margins": 20.727497100830078, |
| "rewards/rejected": -19.167320251464844, |
| "step": 1510 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 2.5621511922881783e-07, |
| "logits/chosen": -2.194122791290283, |
| "logits/rejected": -1.7103767395019531, |
| "logps/chosen": -86.33879089355469, |
| "logps/rejected": -105.571533203125, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.7159111499786377, |
| "rewards/margins": 23.524215698242188, |
| "rewards/rejected": -19.808303833007812, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 2.557077625570776e-07, |
| "logits/chosen": -2.1951489448547363, |
| "logits/rejected": -1.881566047668457, |
| "logps/chosen": -86.02349853515625, |
| "logps/rejected": -113.3117446899414, |
| "loss": 0.0061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.7853310108184814, |
| "rewards/margins": 21.609209060668945, |
| "rewards/rejected": -19.823875427246094, |
| "step": 1530 |
| }, |
| { |
| "epoch": 0.7, |
| "learning_rate": 2.5520040588533736e-07, |
| "logits/chosen": -2.233779191970825, |
| "logits/rejected": -1.821406602859497, |
| "logps/chosen": -89.24073028564453, |
| "logps/rejected": -110.7870101928711, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8013709783554077, |
| "rewards/margins": 23.02773666381836, |
| "rewards/rejected": -21.226367950439453, |
| "step": 1540 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 2.5469304921359715e-07, |
| "logits/chosen": -2.2160837650299072, |
| "logits/rejected": -1.8794729709625244, |
| "logps/chosen": -85.04659271240234, |
| "logps/rejected": -111.51092529296875, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.755566358566284, |
| "rewards/margins": 22.784481048583984, |
| "rewards/rejected": -20.02891731262207, |
| "step": 1550 |
| }, |
| { |
| "epoch": 0.71, |
| "learning_rate": 2.541856925418569e-07, |
| "logits/chosen": -2.1777358055114746, |
| "logits/rejected": -1.827588438987732, |
| "logps/chosen": -83.27925109863281, |
| "logps/rejected": -109.00111389160156, |
| "loss": 0.0064, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.335489511489868, |
| "rewards/margins": 22.536785125732422, |
| "rewards/rejected": -20.201297760009766, |
| "step": 1560 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 2.536783358701167e-07, |
| "logits/chosen": -2.227769374847412, |
| "logits/rejected": -1.8616615533828735, |
| "logps/chosen": -83.50282287597656, |
| "logps/rejected": -108.1055908203125, |
| "loss": 0.0029, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6894235610961914, |
| "rewards/margins": 23.30007553100586, |
| "rewards/rejected": -20.610652923583984, |
| "step": 1570 |
| }, |
| { |
| "epoch": 0.72, |
| "learning_rate": 2.5317097919837643e-07, |
| "logits/chosen": -2.2765088081359863, |
| "logits/rejected": -1.8756946325302124, |
| "logps/chosen": -86.42229461669922, |
| "logps/rejected": -113.13868713378906, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7841994762420654, |
| "rewards/margins": 23.93669319152832, |
| "rewards/rejected": -22.152494430541992, |
| "step": 1580 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 2.526636225266362e-07, |
| "logits/chosen": -2.2187094688415527, |
| "logits/rejected": -1.8117910623550415, |
| "logps/chosen": -90.1899185180664, |
| "logps/rejected": -115.62690734863281, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5328062176704407, |
| "rewards/margins": 23.20620346069336, |
| "rewards/rejected": -22.673397064208984, |
| "step": 1590 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 2.5215626585489596e-07, |
| "logits/chosen": -2.2092316150665283, |
| "logits/rejected": -1.886833906173706, |
| "logps/chosen": -83.89598083496094, |
| "logps/rejected": -113.24908447265625, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.40551048517227173, |
| "rewards/margins": 22.66997528076172, |
| "rewards/rejected": -22.264461517333984, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.73, |
| "eval_logits/chosen": -2.203066110610962, |
| "eval_logits/rejected": -1.853004813194275, |
| "eval_logps/chosen": -87.39132690429688, |
| "eval_logps/rejected": -111.6284408569336, |
| "eval_loss": 0.006391549948602915, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.7196283340454102, |
| "eval_rewards/margins": 23.118391036987305, |
| "eval_rewards/rejected": -22.398759841918945, |
| "eval_runtime": 187.7477, |
| "eval_samples_per_second": 15.244, |
| "eval_steps_per_second": 0.953, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.73, |
| "learning_rate": 2.5164890918315575e-07, |
| "logits/chosen": -2.2956223487854004, |
| "logits/rejected": -1.944941759109497, |
| "logps/chosen": -89.06498718261719, |
| "logps/rejected": -114.08534240722656, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5906304121017456, |
| "rewards/margins": 23.316919326782227, |
| "rewards/rejected": -22.726289749145508, |
| "step": 1610 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 2.511415525114155e-07, |
| "logits/chosen": -2.2368171215057373, |
| "logits/rejected": -1.9505856037139893, |
| "logps/chosen": -89.01737213134766, |
| "logps/rejected": -112.30778503417969, |
| "loss": 0.0038, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.34434378147125244, |
| "rewards/margins": 21.64006996154785, |
| "rewards/rejected": -21.295726776123047, |
| "step": 1620 |
| }, |
| { |
| "epoch": 0.74, |
| "learning_rate": 2.506341958396753e-07, |
| "logits/chosen": -2.1280384063720703, |
| "logits/rejected": -1.7620325088500977, |
| "logps/chosen": -89.35337829589844, |
| "logps/rejected": -113.1550521850586, |
| "loss": 0.0077, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.42981046438217163, |
| "rewards/margins": 23.936683654785156, |
| "rewards/rejected": -23.506874084472656, |
| "step": 1630 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 2.5012683916793503e-07, |
| "logits/chosen": -2.142151355743408, |
| "logits/rejected": -1.7131553888320923, |
| "logps/chosen": -92.11378479003906, |
| "logps/rejected": -114.9017562866211, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4623570442199707, |
| "rewards/margins": 24.9403133392334, |
| "rewards/rejected": -22.47795867919922, |
| "step": 1640 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 2.496194824961948e-07, |
| "logits/chosen": -2.2092690467834473, |
| "logits/rejected": -1.8049709796905518, |
| "logps/chosen": -87.24295806884766, |
| "logps/rejected": -112.48702239990234, |
| "loss": 0.0028, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.603916645050049, |
| "rewards/margins": 24.936174392700195, |
| "rewards/rejected": -22.332256317138672, |
| "step": 1650 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 2.4911212582445456e-07, |
| "logits/chosen": -2.184044599533081, |
| "logits/rejected": -1.7883659601211548, |
| "logps/chosen": -84.82064056396484, |
| "logps/rejected": -112.11551666259766, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.598346710205078, |
| "rewards/margins": 24.60263442993164, |
| "rewards/rejected": -22.004289627075195, |
| "step": 1660 |
| }, |
| { |
| "epoch": 0.76, |
| "learning_rate": 2.4860476915271435e-07, |
| "logits/chosen": -2.2834110260009766, |
| "logits/rejected": -1.8969223499298096, |
| "logps/chosen": -84.3614273071289, |
| "logps/rejected": -114.11153411865234, |
| "loss": 0.0108, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1697537899017334, |
| "rewards/margins": 25.301849365234375, |
| "rewards/rejected": -23.132095336914062, |
| "step": 1670 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 2.480974124809741e-07, |
| "logits/chosen": -2.228867769241333, |
| "logits/rejected": -1.776439905166626, |
| "logps/chosen": -92.27311706542969, |
| "logps/rejected": -115.7362289428711, |
| "loss": 0.0031, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.5189244747161865, |
| "rewards/margins": 23.18109703063965, |
| "rewards/rejected": -20.662174224853516, |
| "step": 1680 |
| }, |
| { |
| "epoch": 0.77, |
| "learning_rate": 2.475900558092339e-07, |
| "logits/chosen": -2.2656030654907227, |
| "logits/rejected": -1.8951988220214844, |
| "logps/chosen": -87.36007690429688, |
| "logps/rejected": -105.27301025390625, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5274977684020996, |
| "rewards/margins": 22.20186424255371, |
| "rewards/rejected": -19.674365997314453, |
| "step": 1690 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 2.4708269913749363e-07, |
| "logits/chosen": -2.2209548950195312, |
| "logits/rejected": -1.8660656213760376, |
| "logps/chosen": -84.46993255615234, |
| "logps/rejected": -111.65545654296875, |
| "loss": 0.0034, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.7445751428604126, |
| "rewards/margins": 22.078369140625, |
| "rewards/rejected": -20.333797454833984, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.78, |
| "eval_logits/chosen": -2.1876187324523926, |
| "eval_logits/rejected": -1.8381489515304565, |
| "eval_logps/chosen": -85.6910171508789, |
| "eval_logps/rejected": -108.63225555419922, |
| "eval_loss": 0.0056876870803534985, |
| "eval_rewards/accuracies": 0.9972066879272461, |
| "eval_rewards/chosen": 1.5697858333587646, |
| "eval_rewards/margins": 22.470449447631836, |
| "eval_rewards/rejected": -20.900663375854492, |
| "eval_runtime": 262.3348, |
| "eval_samples_per_second": 10.91, |
| "eval_steps_per_second": 0.682, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.78, |
| "learning_rate": 2.465753424657534e-07, |
| "logits/chosen": -2.1472651958465576, |
| "logits/rejected": -1.737198829650879, |
| "logps/chosen": -85.45567321777344, |
| "logps/rejected": -110.42171478271484, |
| "loss": 0.0016, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.159968614578247, |
| "rewards/margins": 23.5798397064209, |
| "rewards/rejected": -21.419872283935547, |
| "step": 1710 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 2.4606798579401316e-07, |
| "logits/chosen": -2.1188554763793945, |
| "logits/rejected": -1.82101571559906, |
| "logps/chosen": -85.29725646972656, |
| "logps/rejected": -111.76287841796875, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8330132961273193, |
| "rewards/margins": 21.782207489013672, |
| "rewards/rejected": -19.949195861816406, |
| "step": 1720 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 2.4556062912227295e-07, |
| "logits/chosen": -2.1919398307800293, |
| "logits/rejected": -1.777989149093628, |
| "logps/chosen": -89.14543151855469, |
| "logps/rejected": -109.812744140625, |
| "loss": 0.004, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.5910837650299072, |
| "rewards/margins": 22.191823959350586, |
| "rewards/rejected": -19.60074234008789, |
| "step": 1730 |
| }, |
| { |
| "epoch": 0.79, |
| "learning_rate": 2.450532724505327e-07, |
| "logits/chosen": -2.2214889526367188, |
| "logits/rejected": -1.8154666423797607, |
| "logps/chosen": -84.61921691894531, |
| "logps/rejected": -109.73350524902344, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.7251133918762207, |
| "rewards/margins": 23.308391571044922, |
| "rewards/rejected": -20.58327865600586, |
| "step": 1740 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 2.445459157787925e-07, |
| "logits/chosen": -2.266470432281494, |
| "logits/rejected": -1.869768500328064, |
| "logps/chosen": -90.2508316040039, |
| "logps/rejected": -113.48309326171875, |
| "loss": 0.0054, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.311676502227783, |
| "rewards/margins": 23.228078842163086, |
| "rewards/rejected": -20.916400909423828, |
| "step": 1750 |
| }, |
| { |
| "epoch": 0.8, |
| "learning_rate": 2.4403855910705223e-07, |
| "logits/chosen": -2.214456081390381, |
| "logits/rejected": -1.8925609588623047, |
| "logps/chosen": -82.62592315673828, |
| "logps/rejected": -108.47261047363281, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.60259211063385, |
| "rewards/margins": 21.090885162353516, |
| "rewards/rejected": -19.488292694091797, |
| "step": 1760 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 2.43531202435312e-07, |
| "logits/chosen": -2.18231463432312, |
| "logits/rejected": -1.7262542247772217, |
| "logps/chosen": -94.32820892333984, |
| "logps/rejected": -114.97127532958984, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.17089581489563, |
| "rewards/margins": 24.209266662597656, |
| "rewards/rejected": -22.038372039794922, |
| "step": 1770 |
| }, |
| { |
| "epoch": 0.81, |
| "learning_rate": 2.4302384576357176e-07, |
| "logits/chosen": -2.221064329147339, |
| "logits/rejected": -1.8696062564849854, |
| "logps/chosen": -90.65652465820312, |
| "logps/rejected": -123.33599853515625, |
| "loss": 0.0012, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8242883682250977, |
| "rewards/margins": 25.92755126953125, |
| "rewards/rejected": -24.103261947631836, |
| "step": 1780 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 2.4251648909183155e-07, |
| "logits/chosen": -2.2197823524475098, |
| "logits/rejected": -1.8690903186798096, |
| "logps/chosen": -87.0594482421875, |
| "logps/rejected": -113.86370849609375, |
| "loss": 0.0062, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.1697230339050293, |
| "rewards/margins": 24.501766204833984, |
| "rewards/rejected": -21.332040786743164, |
| "step": 1790 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 2.420091324200913e-07, |
| "logits/chosen": -2.231548309326172, |
| "logits/rejected": -1.9493480920791626, |
| "logps/chosen": -86.16682434082031, |
| "logps/rejected": -116.98976135253906, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1170125007629395, |
| "rewards/margins": 23.978233337402344, |
| "rewards/rejected": -22.861225128173828, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.82, |
| "eval_logits/chosen": -2.1833367347717285, |
| "eval_logits/rejected": -1.8318486213684082, |
| "eval_logps/chosen": -85.54547119140625, |
| "eval_logps/rejected": -111.36595153808594, |
| "eval_loss": 0.005670672748237848, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 1.6425559520721436, |
| "eval_rewards/margins": 23.910072326660156, |
| "eval_rewards/rejected": -22.267513275146484, |
| "eval_runtime": 194.2178, |
| "eval_samples_per_second": 14.736, |
| "eval_steps_per_second": 0.922, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 2.415017757483511e-07, |
| "logits/chosen": -2.2005953788757324, |
| "logits/rejected": -1.7761281728744507, |
| "logps/chosen": -94.50230407714844, |
| "logps/rejected": -119.65816497802734, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.642756700515747, |
| "rewards/margins": 27.622684478759766, |
| "rewards/rejected": -24.979928970336914, |
| "step": 1810 |
| }, |
| { |
| "epoch": 0.83, |
| "learning_rate": 2.409944190766108e-07, |
| "logits/chosen": -2.2361111640930176, |
| "logits/rejected": -1.850996732711792, |
| "logps/chosen": -85.87068176269531, |
| "logps/rejected": -119.11030578613281, |
| "loss": 0.0047, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5628911852836609, |
| "rewards/margins": 25.55295181274414, |
| "rewards/rejected": -24.99005699157715, |
| "step": 1820 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 2.404870624048706e-07, |
| "logits/chosen": -2.179274559020996, |
| "logits/rejected": -1.809653878211975, |
| "logps/chosen": -83.79485321044922, |
| "logps/rejected": -107.97098541259766, |
| "loss": 0.0061, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3543555736541748, |
| "rewards/margins": 23.6602840423584, |
| "rewards/rejected": -22.305927276611328, |
| "step": 1830 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 2.3997970573313036e-07, |
| "logits/chosen": -2.2063956260681152, |
| "logits/rejected": -1.8312809467315674, |
| "logps/chosen": -82.93327331542969, |
| "logps/rejected": -108.3280029296875, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.580742835998535, |
| "rewards/margins": 24.954925537109375, |
| "rewards/rejected": -22.374181747436523, |
| "step": 1840 |
| }, |
| { |
| "epoch": 0.84, |
| "learning_rate": 2.3947234906139015e-07, |
| "logits/chosen": -2.2583651542663574, |
| "logits/rejected": -1.8435817956924438, |
| "logps/chosen": -87.89311981201172, |
| "logps/rejected": -114.3637466430664, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6345696449279785, |
| "rewards/margins": 26.21219825744629, |
| "rewards/rejected": -23.577627182006836, |
| "step": 1850 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 2.389649923896499e-07, |
| "logits/chosen": -2.1602165699005127, |
| "logits/rejected": -1.8217239379882812, |
| "logps/chosen": -86.65802001953125, |
| "logps/rejected": -116.1138916015625, |
| "loss": 0.005, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.0264811515808105, |
| "rewards/margins": 24.80233383178711, |
| "rewards/rejected": -21.77585220336914, |
| "step": 1860 |
| }, |
| { |
| "epoch": 0.85, |
| "learning_rate": 2.384576357179097e-07, |
| "logits/chosen": -2.1380181312561035, |
| "logits/rejected": -1.7056655883789062, |
| "logps/chosen": -90.73863220214844, |
| "logps/rejected": -117.328125, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.132702350616455, |
| "rewards/margins": 26.685169219970703, |
| "rewards/rejected": -23.552465438842773, |
| "step": 1870 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.3795027904616943e-07, |
| "logits/chosen": -2.2756571769714355, |
| "logits/rejected": -1.9350929260253906, |
| "logps/chosen": -90.4540786743164, |
| "logps/rejected": -118.94252014160156, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2916797399520874, |
| "rewards/margins": 23.93198013305664, |
| "rewards/rejected": -22.640300750732422, |
| "step": 1880 |
| }, |
| { |
| "epoch": 0.86, |
| "learning_rate": 2.374429223744292e-07, |
| "logits/chosen": -2.2293753623962402, |
| "logits/rejected": -1.8584178686141968, |
| "logps/chosen": -83.38288879394531, |
| "logps/rejected": -109.228759765625, |
| "loss": 0.0038, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3209164142608643, |
| "rewards/margins": 24.548112869262695, |
| "rewards/rejected": -22.227197647094727, |
| "step": 1890 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.3693556570268896e-07, |
| "logits/chosen": -2.28892183303833, |
| "logits/rejected": -1.9423027038574219, |
| "logps/chosen": -86.03301239013672, |
| "logps/rejected": -118.62422180175781, |
| "loss": 0.0038, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0244522094726562, |
| "rewards/margins": 25.199527740478516, |
| "rewards/rejected": -23.17507553100586, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.87, |
| "eval_logits/chosen": -2.2072012424468994, |
| "eval_logits/rejected": -1.8581523895263672, |
| "eval_logps/chosen": -85.27046203613281, |
| "eval_logps/rejected": -113.4885025024414, |
| "eval_loss": 0.006078703328967094, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.780059814453125, |
| "eval_rewards/margins": 25.108850479125977, |
| "eval_rewards/rejected": -23.32879066467285, |
| "eval_runtime": 195.9406, |
| "eval_samples_per_second": 14.606, |
| "eval_steps_per_second": 0.914, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.87, |
| "learning_rate": 2.3642820903094873e-07, |
| "logits/chosen": -2.218658208847046, |
| "logits/rejected": -1.8969409465789795, |
| "logps/chosen": -82.16979217529297, |
| "logps/rejected": -116.5958023071289, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.3920204639434814, |
| "rewards/margins": 26.6087646484375, |
| "rewards/rejected": -24.21674156188965, |
| "step": 1910 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.359208523592085e-07, |
| "logits/chosen": -2.2358078956604004, |
| "logits/rejected": -1.8991063833236694, |
| "logps/chosen": -85.44503021240234, |
| "logps/rejected": -115.95997619628906, |
| "loss": 0.0071, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.7518513202667236, |
| "rewards/margins": 25.580867767333984, |
| "rewards/rejected": -23.829017639160156, |
| "step": 1920 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.3541349568746826e-07, |
| "logits/chosen": -2.23246169090271, |
| "logits/rejected": -1.8101627826690674, |
| "logps/chosen": -88.6487045288086, |
| "logps/rejected": -115.31546783447266, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0985748767852783, |
| "rewards/margins": 25.528507232666016, |
| "rewards/rejected": -23.429927825927734, |
| "step": 1930 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 2.3490613901572803e-07, |
| "logits/chosen": -2.1937997341156006, |
| "logits/rejected": -1.899291753768921, |
| "logps/chosen": -82.33692169189453, |
| "logps/rejected": -116.04981994628906, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9832813739776611, |
| "rewards/margins": 25.707172393798828, |
| "rewards/rejected": -23.72389030456543, |
| "step": 1940 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 2.343987823439878e-07, |
| "logits/chosen": -2.19626522064209, |
| "logits/rejected": -1.8097482919692993, |
| "logps/chosen": -81.79765319824219, |
| "logps/rejected": -115.43753814697266, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6907103061676025, |
| "rewards/margins": 26.663219451904297, |
| "rewards/rejected": -23.972511291503906, |
| "step": 1950 |
| }, |
| { |
| "epoch": 0.89, |
| "learning_rate": 2.3389142567224756e-07, |
| "logits/chosen": -2.2777841091156006, |
| "logits/rejected": -1.932579755783081, |
| "logps/chosen": -93.16586303710938, |
| "logps/rejected": -125.8462905883789, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5280853509902954, |
| "rewards/margins": 27.09100914001465, |
| "rewards/rejected": -25.562923431396484, |
| "step": 1960 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 2.3338406900050733e-07, |
| "logits/chosen": -2.2580790519714355, |
| "logits/rejected": -1.8621156215667725, |
| "logps/chosen": -85.24269104003906, |
| "logps/rejected": -114.29624938964844, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.0481958389282227, |
| "rewards/margins": 28.0736141204834, |
| "rewards/rejected": -25.025419235229492, |
| "step": 1970 |
| }, |
| { |
| "epoch": 0.9, |
| "learning_rate": 2.328767123287671e-07, |
| "logits/chosen": -2.237919569015503, |
| "logits/rejected": -1.9384901523590088, |
| "logps/chosen": -85.32179260253906, |
| "logps/rejected": -126.17274475097656, |
| "loss": 0.0019, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1654307842254639, |
| "rewards/margins": 29.019460678100586, |
| "rewards/rejected": -27.854028701782227, |
| "step": 1980 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 2.3236935565702686e-07, |
| "logits/chosen": -2.2402701377868652, |
| "logits/rejected": -1.9076135158538818, |
| "logps/chosen": -84.37796783447266, |
| "logps/rejected": -119.68265533447266, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9258636236190796, |
| "rewards/margins": 26.976810455322266, |
| "rewards/rejected": -25.050945281982422, |
| "step": 1990 |
| }, |
| { |
| "epoch": 0.91, |
| "learning_rate": 2.3186199898528663e-07, |
| "logits/chosen": -2.2411704063415527, |
| "logits/rejected": -1.8752963542938232, |
| "logps/chosen": -85.16765594482422, |
| "logps/rejected": -116.25230407714844, |
| "loss": 0.0074, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9921166896820068, |
| "rewards/margins": 26.783111572265625, |
| "rewards/rejected": -24.790996551513672, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.91, |
| "eval_logits/chosen": -2.1969516277313232, |
| "eval_logits/rejected": -1.849937081336975, |
| "eval_logps/chosen": -86.06404113769531, |
| "eval_logps/rejected": -117.32756042480469, |
| "eval_loss": 0.006207953207194805, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.3832741975784302, |
| "eval_rewards/margins": 26.631595611572266, |
| "eval_rewards/rejected": -25.24832534790039, |
| "eval_runtime": 285.1562, |
| "eval_samples_per_second": 10.037, |
| "eval_steps_per_second": 0.628, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 2.313546423135464e-07, |
| "logits/chosen": -2.1540346145629883, |
| "logits/rejected": -1.7665355205535889, |
| "logps/chosen": -90.4669189453125, |
| "logps/rejected": -113.56608581542969, |
| "loss": 0.0323, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6254875659942627, |
| "rewards/margins": 24.596233367919922, |
| "rewards/rejected": -21.970745086669922, |
| "step": 2010 |
| }, |
| { |
| "epoch": 0.92, |
| "learning_rate": 2.3084728564180616e-07, |
| "logits/chosen": -2.095485210418701, |
| "logits/rejected": -1.8191229104995728, |
| "logps/chosen": -82.52064514160156, |
| "logps/rejected": -114.91255187988281, |
| "loss": 0.0007, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7258262634277344, |
| "rewards/margins": 23.092130661010742, |
| "rewards/rejected": -20.366304397583008, |
| "step": 2020 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 2.3033992897006593e-07, |
| "logits/chosen": -2.269178867340088, |
| "logits/rejected": -1.8472541570663452, |
| "logps/chosen": -85.8851318359375, |
| "logps/rejected": -109.74867248535156, |
| "loss": 0.0059, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9320855140686035, |
| "rewards/margins": 24.42289161682129, |
| "rewards/rejected": -21.49080467224121, |
| "step": 2030 |
| }, |
| { |
| "epoch": 0.93, |
| "learning_rate": 2.298325722983257e-07, |
| "logits/chosen": -2.1464996337890625, |
| "logits/rejected": -1.7602676153182983, |
| "logps/chosen": -87.7806625366211, |
| "logps/rejected": -114.57737731933594, |
| "loss": 0.0039, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2257797718048096, |
| "rewards/margins": 24.386262893676758, |
| "rewards/rejected": -22.160480499267578, |
| "step": 2040 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 2.2932521562658546e-07, |
| "logits/chosen": -2.1523966789245605, |
| "logits/rejected": -1.8064305782318115, |
| "logps/chosen": -86.0357666015625, |
| "logps/rejected": -112.4377212524414, |
| "loss": 0.01, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2664999961853027, |
| "rewards/margins": 24.120498657226562, |
| "rewards/rejected": -21.853994369506836, |
| "step": 2050 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 2.2881785895484523e-07, |
| "logits/chosen": -2.209660291671753, |
| "logits/rejected": -1.8102290630340576, |
| "logps/chosen": -87.49055480957031, |
| "logps/rejected": -110.42215728759766, |
| "loss": 0.0084, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9767696857452393, |
| "rewards/margins": 23.30925941467285, |
| "rewards/rejected": -20.33249282836914, |
| "step": 2060 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 2.28310502283105e-07, |
| "logits/chosen": -2.1463191509246826, |
| "logits/rejected": -1.8173093795776367, |
| "logps/chosen": -80.68872833251953, |
| "logps/rejected": -112.73249816894531, |
| "loss": 0.0017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.864393711090088, |
| "rewards/margins": 23.488155364990234, |
| "rewards/rejected": -20.623760223388672, |
| "step": 2070 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 2.2780314561136476e-07, |
| "logits/chosen": -2.2494287490844727, |
| "logits/rejected": -1.8683185577392578, |
| "logps/chosen": -90.07569885253906, |
| "logps/rejected": -109.84877014160156, |
| "loss": 0.0027, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.9630918502807617, |
| "rewards/margins": 24.1021728515625, |
| "rewards/rejected": -21.139080047607422, |
| "step": 2080 |
| }, |
| { |
| "epoch": 0.95, |
| "learning_rate": 2.2729578893962453e-07, |
| "logits/chosen": -2.2044570446014404, |
| "logits/rejected": -1.8658645153045654, |
| "logps/chosen": -87.5743179321289, |
| "logps/rejected": -112.40694427490234, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.718559741973877, |
| "rewards/margins": 23.71183967590332, |
| "rewards/rejected": -20.9932804107666, |
| "step": 2090 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.267884322678843e-07, |
| "logits/chosen": -2.25122332572937, |
| "logits/rejected": -1.8896070718765259, |
| "logps/chosen": -85.34817504882812, |
| "logps/rejected": -111.40423583984375, |
| "loss": 0.0052, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9988149404525757, |
| "rewards/margins": 24.812755584716797, |
| "rewards/rejected": -22.813940048217773, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.96, |
| "eval_logits/chosen": -2.1680314540863037, |
| "eval_logits/rejected": -1.8186105489730835, |
| "eval_logps/chosen": -85.12017822265625, |
| "eval_logps/rejected": -110.73912048339844, |
| "eval_loss": 0.005591261200606823, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 1.8552027940750122, |
| "eval_rewards/margins": 23.80930519104004, |
| "eval_rewards/rejected": -21.954099655151367, |
| "eval_runtime": 178.0339, |
| "eval_samples_per_second": 16.076, |
| "eval_steps_per_second": 1.005, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.96, |
| "learning_rate": 2.2628107559614406e-07, |
| "logits/chosen": -2.182180643081665, |
| "logits/rejected": -1.887372612953186, |
| "logps/chosen": -84.8567123413086, |
| "logps/rejected": -115.8074722290039, |
| "loss": 0.0061, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.302744150161743, |
| "rewards/margins": 24.386987686157227, |
| "rewards/rejected": -22.084239959716797, |
| "step": 2110 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 2.2577371892440383e-07, |
| "logits/chosen": -2.1381964683532715, |
| "logits/rejected": -1.7454639673233032, |
| "logps/chosen": -89.93260192871094, |
| "logps/rejected": -116.71270751953125, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.476393938064575, |
| "rewards/margins": 26.676769256591797, |
| "rewards/rejected": -24.200376510620117, |
| "step": 2120 |
| }, |
| { |
| "epoch": 0.97, |
| "learning_rate": 2.252663622526636e-07, |
| "logits/chosen": -2.1717324256896973, |
| "logits/rejected": -1.7591243982315063, |
| "logps/chosen": -92.45228576660156, |
| "logps/rejected": -119.2745361328125, |
| "loss": 0.0051, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9234495162963867, |
| "rewards/margins": 26.036029815673828, |
| "rewards/rejected": -24.112579345703125, |
| "step": 2130 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 2.2475900558092336e-07, |
| "logits/chosen": -2.2273201942443848, |
| "logits/rejected": -1.8839191198349, |
| "logps/chosen": -82.78079223632812, |
| "logps/rejected": -117.9035415649414, |
| "loss": 0.002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3843512535095215, |
| "rewards/margins": 26.67987632751465, |
| "rewards/rejected": -24.2955265045166, |
| "step": 2140 |
| }, |
| { |
| "epoch": 0.98, |
| "learning_rate": 2.2425164890918313e-07, |
| "logits/chosen": -2.1497018337249756, |
| "logits/rejected": -1.7351865768432617, |
| "logps/chosen": -91.68416595458984, |
| "logps/rejected": -117.03997802734375, |
| "loss": 0.004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7206225395202637, |
| "rewards/margins": 25.490280151367188, |
| "rewards/rejected": -22.7696590423584, |
| "step": 2150 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 2.237442922374429e-07, |
| "logits/chosen": -2.105646848678589, |
| "logits/rejected": -1.7680670022964478, |
| "logps/chosen": -83.17054748535156, |
| "logps/rejected": -110.3686294555664, |
| "loss": 0.003, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5340194702148438, |
| "rewards/margins": 23.920883178710938, |
| "rewards/rejected": -22.386865615844727, |
| "step": 2160 |
| }, |
| { |
| "epoch": 0.99, |
| "learning_rate": 2.2323693556570266e-07, |
| "logits/chosen": -2.165457248687744, |
| "logits/rejected": -1.7284084558486938, |
| "logps/chosen": -87.86439514160156, |
| "logps/rejected": -118.21602630615234, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8017003536224365, |
| "rewards/margins": 25.351261138916016, |
| "rewards/rejected": -22.549560546875, |
| "step": 2170 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 2.2272957889396242e-07, |
| "logits/chosen": -2.1325504779815674, |
| "logits/rejected": -1.746701955795288, |
| "logps/chosen": -86.5936279296875, |
| "logps/rejected": -111.45501708984375, |
| "loss": 0.0061, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3284072875976562, |
| "rewards/margins": 23.549602508544922, |
| "rewards/rejected": -21.221195220947266, |
| "step": 2180 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 2.222222222222222e-07, |
| "logits/chosen": -2.1646571159362793, |
| "logits/rejected": -1.8008124828338623, |
| "logps/chosen": -91.38953399658203, |
| "logps/rejected": -111.65739440917969, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5468361377716064, |
| "rewards/margins": 23.903175354003906, |
| "rewards/rejected": -21.356340408325195, |
| "step": 2190 |
| }, |
| { |
| "epoch": 1.0, |
| "learning_rate": 2.2171486555048196e-07, |
| "logits/chosen": -2.1863300800323486, |
| "logits/rejected": -1.8556190729141235, |
| "logps/chosen": -88.28529357910156, |
| "logps/rejected": -117.9306869506836, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.728421926498413, |
| "rewards/margins": 25.30516242980957, |
| "rewards/rejected": -22.57674217224121, |
| "step": 2200 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_logits/chosen": -2.1533539295196533, |
| "eval_logits/rejected": -1.8051024675369263, |
| "eval_logps/chosen": -85.23373413085938, |
| "eval_logps/rejected": -109.83948516845703, |
| "eval_loss": 0.005602886434644461, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 1.7984265089035034, |
| "eval_rewards/margins": 23.30270767211914, |
| "eval_rewards/rejected": -21.50428009033203, |
| "eval_runtime": 199.8494, |
| "eval_samples_per_second": 14.321, |
| "eval_steps_per_second": 0.896, |
| "step": 2200 |
| }, |
| { |
| "epoch": 1.01, |
| "learning_rate": 2.2120750887874172e-07, |
| "logits/chosen": -2.184231996536255, |
| "logits/rejected": -1.8242343664169312, |
| "logps/chosen": -87.5803451538086, |
| "logps/rejected": -116.75703430175781, |
| "loss": 0.0047, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.086021900177002, |
| "rewards/margins": 24.399898529052734, |
| "rewards/rejected": -21.31387710571289, |
| "step": 2210 |
| }, |
| { |
| "epoch": 1.01, |
| "learning_rate": 2.207001522070015e-07, |
| "logits/chosen": -2.141301393508911, |
| "logits/rejected": -1.7759917974472046, |
| "logps/chosen": -91.86133575439453, |
| "logps/rejected": -116.03465270996094, |
| "loss": 0.0027, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5139381885528564, |
| "rewards/margins": 24.87776756286621, |
| "rewards/rejected": -22.363828659057617, |
| "step": 2220 |
| }, |
| { |
| "epoch": 1.02, |
| "learning_rate": 2.2019279553526126e-07, |
| "logits/chosen": -2.147770643234253, |
| "logits/rejected": -1.6709057092666626, |
| "logps/chosen": -95.53349304199219, |
| "logps/rejected": -112.99432373046875, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.351959705352783, |
| "rewards/margins": 23.853574752807617, |
| "rewards/rejected": -21.501617431640625, |
| "step": 2230 |
| }, |
| { |
| "epoch": 1.02, |
| "learning_rate": 2.1968543886352102e-07, |
| "logits/chosen": -2.23237943649292, |
| "logits/rejected": -1.9411084651947021, |
| "logps/chosen": -82.67511749267578, |
| "logps/rejected": -113.44953918457031, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2467036247253418, |
| "rewards/margins": 23.66016960144043, |
| "rewards/rejected": -22.413467407226562, |
| "step": 2240 |
| }, |
| { |
| "epoch": 1.03, |
| "learning_rate": 2.191780821917808e-07, |
| "logits/chosen": -2.1674644947052, |
| "logits/rejected": -1.8577144145965576, |
| "logps/chosen": -84.78733825683594, |
| "logps/rejected": -115.30723571777344, |
| "loss": 0.0019, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1725082397460938, |
| "rewards/margins": 24.452693939208984, |
| "rewards/rejected": -22.280183792114258, |
| "step": 2250 |
| }, |
| { |
| "epoch": 1.03, |
| "learning_rate": 2.1867072552004056e-07, |
| "logits/chosen": -2.288512706756592, |
| "logits/rejected": -1.899009108543396, |
| "logps/chosen": -87.27893829345703, |
| "logps/rejected": -116.2738265991211, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.6636362075805664, |
| "rewards/margins": 25.80401039123535, |
| "rewards/rejected": -23.1403751373291, |
| "step": 2260 |
| }, |
| { |
| "epoch": 1.04, |
| "learning_rate": 2.1816336884830032e-07, |
| "logits/chosen": -2.1499955654144287, |
| "logits/rejected": -1.848623514175415, |
| "logps/chosen": -81.82324981689453, |
| "logps/rejected": -115.3056411743164, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2145495414733887, |
| "rewards/margins": 25.514049530029297, |
| "rewards/rejected": -23.299501419067383, |
| "step": 2270 |
| }, |
| { |
| "epoch": 1.04, |
| "learning_rate": 2.176560121765601e-07, |
| "logits/chosen": -2.2483465671539307, |
| "logits/rejected": -1.9416240453720093, |
| "logps/chosen": -87.2221908569336, |
| "logps/rejected": -114.78794860839844, |
| "loss": 0.0068, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0889229774475098, |
| "rewards/margins": 24.115562438964844, |
| "rewards/rejected": -22.026639938354492, |
| "step": 2280 |
| }, |
| { |
| "epoch": 1.05, |
| "learning_rate": 2.1714865550481986e-07, |
| "logits/chosen": -2.1781678199768066, |
| "logits/rejected": -1.788074254989624, |
| "logps/chosen": -85.21916961669922, |
| "logps/rejected": -115.5194091796875, |
| "loss": 0.0056, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.176715850830078, |
| "rewards/margins": 25.536880493164062, |
| "rewards/rejected": -23.360164642333984, |
| "step": 2290 |
| }, |
| { |
| "epoch": 1.05, |
| "learning_rate": 2.1664129883307962e-07, |
| "logits/chosen": -2.1438374519348145, |
| "logits/rejected": -1.81368887424469, |
| "logps/chosen": -84.81913757324219, |
| "logps/rejected": -114.127685546875, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9444429874420166, |
| "rewards/margins": 24.719940185546875, |
| "rewards/rejected": -22.775497436523438, |
| "step": 2300 |
| }, |
| { |
| "epoch": 1.05, |
| "eval_logits/chosen": -2.1615262031555176, |
| "eval_logits/rejected": -1.812113881111145, |
| "eval_logps/chosen": -85.03972625732422, |
| "eval_logps/rejected": -110.49690246582031, |
| "eval_loss": 0.005681305192410946, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.8954312801361084, |
| "eval_rewards/margins": 23.728422164916992, |
| "eval_rewards/rejected": -21.832990646362305, |
| "eval_runtime": 213.6661, |
| "eval_samples_per_second": 13.395, |
| "eval_steps_per_second": 0.838, |
| "step": 2300 |
| }, |
| { |
| "epoch": 1.05, |
| "learning_rate": 2.161339421613394e-07, |
| "logits/chosen": -2.1927437782287598, |
| "logits/rejected": -1.8735787868499756, |
| "logps/chosen": -85.47551727294922, |
| "logps/rejected": -118.7635269165039, |
| "loss": 0.0014, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.551397681236267, |
| "rewards/margins": 23.236629486083984, |
| "rewards/rejected": -21.685232162475586, |
| "step": 2310 |
| }, |
| { |
| "epoch": 1.06, |
| "learning_rate": 2.1562658548959916e-07, |
| "logits/chosen": -2.259660482406616, |
| "logits/rejected": -1.8416106700897217, |
| "logps/chosen": -88.98851013183594, |
| "logps/rejected": -113.52168273925781, |
| "loss": 0.0056, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8511302471160889, |
| "rewards/margins": 26.2247257232666, |
| "rewards/rejected": -24.37359619140625, |
| "step": 2320 |
| }, |
| { |
| "epoch": 1.06, |
| "learning_rate": 2.1511922881785892e-07, |
| "logits/chosen": -2.151369571685791, |
| "logits/rejected": -1.7967971563339233, |
| "logps/chosen": -86.92037200927734, |
| "logps/rejected": -110.92555236816406, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9498761892318726, |
| "rewards/margins": 22.68193244934082, |
| "rewards/rejected": -20.7320556640625, |
| "step": 2330 |
| }, |
| { |
| "epoch": 1.07, |
| "learning_rate": 2.146118721461187e-07, |
| "logits/chosen": -2.1439459323883057, |
| "logits/rejected": -1.8120663166046143, |
| "logps/chosen": -85.99787902832031, |
| "logps/rejected": -114.73786926269531, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.070859432220459, |
| "rewards/margins": 24.2840518951416, |
| "rewards/rejected": -21.213193893432617, |
| "step": 2340 |
| }, |
| { |
| "epoch": 1.07, |
| "learning_rate": 2.1410451547437846e-07, |
| "logits/chosen": -2.2162632942199707, |
| "logits/rejected": -1.8168309926986694, |
| "logps/chosen": -86.9716796875, |
| "logps/rejected": -110.72579193115234, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.628699541091919, |
| "rewards/margins": 23.844585418701172, |
| "rewards/rejected": -21.215885162353516, |
| "step": 2350 |
| }, |
| { |
| "epoch": 1.08, |
| "learning_rate": 2.1359715880263822e-07, |
| "logits/chosen": -2.174852132797241, |
| "logits/rejected": -1.8075027465820312, |
| "logps/chosen": -83.8674087524414, |
| "logps/rejected": -112.02839660644531, |
| "loss": 0.0035, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.375577926635742, |
| "rewards/margins": 24.027584075927734, |
| "rewards/rejected": -21.65200424194336, |
| "step": 2360 |
| }, |
| { |
| "epoch": 1.08, |
| "learning_rate": 2.13089802130898e-07, |
| "logits/chosen": -2.167109251022339, |
| "logits/rejected": -1.8096405267715454, |
| "logps/chosen": -84.7580337524414, |
| "logps/rejected": -117.7773208618164, |
| "loss": 0.0031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1797847747802734, |
| "rewards/margins": 25.387409210205078, |
| "rewards/rejected": -23.207622528076172, |
| "step": 2370 |
| }, |
| { |
| "epoch": 1.09, |
| "learning_rate": 2.1258244545915776e-07, |
| "logits/chosen": -2.141667127609253, |
| "logits/rejected": -1.7658653259277344, |
| "logps/chosen": -83.34464263916016, |
| "logps/rejected": -108.3573989868164, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8713493347167969, |
| "rewards/margins": 23.396373748779297, |
| "rewards/rejected": -21.525026321411133, |
| "step": 2380 |
| }, |
| { |
| "epoch": 1.09, |
| "learning_rate": 2.1207508878741752e-07, |
| "logits/chosen": -2.1764111518859863, |
| "logits/rejected": -1.8196337223052979, |
| "logps/chosen": -89.48072814941406, |
| "logps/rejected": -112.5357437133789, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0725154876708984, |
| "rewards/margins": 23.079078674316406, |
| "rewards/rejected": -22.006563186645508, |
| "step": 2390 |
| }, |
| { |
| "epoch": 1.1, |
| "learning_rate": 2.115677321156773e-07, |
| "logits/chosen": -2.1829419136047363, |
| "logits/rejected": -1.830394983291626, |
| "logps/chosen": -85.89895629882812, |
| "logps/rejected": -116.04026794433594, |
| "loss": 0.0056, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8246675729751587, |
| "rewards/margins": 24.702465057373047, |
| "rewards/rejected": -22.877796173095703, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.1, |
| "eval_logits/chosen": -2.16304874420166, |
| "eval_logits/rejected": -1.8152433633804321, |
| "eval_logps/chosen": -86.18814086914062, |
| "eval_logps/rejected": -112.2934341430664, |
| "eval_loss": 0.005279215984046459, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 1.3212203979492188, |
| "eval_rewards/margins": 24.052478790283203, |
| "eval_rewards/rejected": -22.731260299682617, |
| "eval_runtime": 219.653, |
| "eval_samples_per_second": 13.03, |
| "eval_steps_per_second": 0.815, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.1, |
| "learning_rate": 2.1106037544393706e-07, |
| "logits/chosen": -2.186826705932617, |
| "logits/rejected": -1.8132612705230713, |
| "logps/chosen": -87.9347152709961, |
| "logps/rejected": -116.61759185791016, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.319627285003662, |
| "rewards/margins": 25.145320892333984, |
| "rewards/rejected": -22.82569122314453, |
| "step": 2410 |
| }, |
| { |
| "epoch": 1.1, |
| "learning_rate": 2.1055301877219682e-07, |
| "logits/chosen": -2.2042155265808105, |
| "logits/rejected": -1.7962379455566406, |
| "logps/chosen": -89.61426544189453, |
| "logps/rejected": -119.98515319824219, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1328256130218506, |
| "rewards/margins": 25.84686279296875, |
| "rewards/rejected": -23.714035034179688, |
| "step": 2420 |
| }, |
| { |
| "epoch": 1.11, |
| "learning_rate": 2.100456621004566e-07, |
| "logits/chosen": -2.12410831451416, |
| "logits/rejected": -1.81709885597229, |
| "logps/chosen": -83.92530822753906, |
| "logps/rejected": -115.90692138671875, |
| "loss": 0.004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.454441785812378, |
| "rewards/margins": 25.90082359313965, |
| "rewards/rejected": -23.44638442993164, |
| "step": 2430 |
| }, |
| { |
| "epoch": 1.11, |
| "learning_rate": 2.0953830542871636e-07, |
| "logits/chosen": -2.24179744720459, |
| "logits/rejected": -1.9268741607666016, |
| "logps/chosen": -83.57841491699219, |
| "logps/rejected": -115.2148666381836, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5743858814239502, |
| "rewards/margins": 24.534379959106445, |
| "rewards/rejected": -22.95999526977539, |
| "step": 2440 |
| }, |
| { |
| "epoch": 1.12, |
| "learning_rate": 2.0903094875697612e-07, |
| "logits/chosen": -2.1769909858703613, |
| "logits/rejected": -1.7283875942230225, |
| "logps/chosen": -94.74465942382812, |
| "logps/rejected": -120.26570129394531, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9194726943969727, |
| "rewards/margins": 25.821313858032227, |
| "rewards/rejected": -23.901838302612305, |
| "step": 2450 |
| }, |
| { |
| "epoch": 1.12, |
| "learning_rate": 2.085235920852359e-07, |
| "logits/chosen": -2.1689000129699707, |
| "logits/rejected": -1.8665683269500732, |
| "logps/chosen": -85.41389465332031, |
| "logps/rejected": -120.8945083618164, |
| "loss": 0.0037, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.879907250404358, |
| "rewards/margins": 26.468700408935547, |
| "rewards/rejected": -24.588794708251953, |
| "step": 2460 |
| }, |
| { |
| "epoch": 1.13, |
| "learning_rate": 2.0801623541349566e-07, |
| "logits/chosen": -2.243333101272583, |
| "logits/rejected": -1.8705765008926392, |
| "logps/chosen": -81.45762634277344, |
| "logps/rejected": -111.65384674072266, |
| "loss": 0.0009, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.552882432937622, |
| "rewards/margins": 25.76004981994629, |
| "rewards/rejected": -23.207164764404297, |
| "step": 2470 |
| }, |
| { |
| "epoch": 1.13, |
| "learning_rate": 2.0750887874175542e-07, |
| "logits/chosen": -2.171480655670166, |
| "logits/rejected": -1.8349215984344482, |
| "logps/chosen": -86.48908996582031, |
| "logps/rejected": -120.96435546875, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6669495105743408, |
| "rewards/margins": 25.430204391479492, |
| "rewards/rejected": -23.763256072998047, |
| "step": 2480 |
| }, |
| { |
| "epoch": 1.14, |
| "learning_rate": 2.070015220700152e-07, |
| "logits/chosen": -2.1459603309631348, |
| "logits/rejected": -1.7423560619354248, |
| "logps/chosen": -87.96595764160156, |
| "logps/rejected": -120.45890045166016, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2795779705047607, |
| "rewards/margins": 26.35489273071289, |
| "rewards/rejected": -24.0753116607666, |
| "step": 2490 |
| }, |
| { |
| "epoch": 1.14, |
| "learning_rate": 2.0649416539827496e-07, |
| "logits/chosen": -2.1519787311553955, |
| "logits/rejected": -1.7902311086654663, |
| "logps/chosen": -85.25035095214844, |
| "logps/rejected": -112.2745590209961, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.7297900915145874, |
| "rewards/margins": 22.654048919677734, |
| "rewards/rejected": -20.92425537109375, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.14, |
| "eval_logits/chosen": -2.1591501235961914, |
| "eval_logits/rejected": -1.8124133348464966, |
| "eval_logps/chosen": -84.112548828125, |
| "eval_logps/rejected": -110.20504760742188, |
| "eval_loss": 0.005225938744843006, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 2.3590192794799805, |
| "eval_rewards/margins": 24.04608726501465, |
| "eval_rewards/rejected": -21.68706703186035, |
| "eval_runtime": 233.6775, |
| "eval_samples_per_second": 12.248, |
| "eval_steps_per_second": 0.766, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.15, |
| "learning_rate": 2.0598680872653472e-07, |
| "logits/chosen": -2.1743123531341553, |
| "logits/rejected": -1.7647409439086914, |
| "logps/chosen": -90.07218933105469, |
| "logps/rejected": -115.3643798828125, |
| "loss": 0.0037, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.752506732940674, |
| "rewards/margins": 24.845409393310547, |
| "rewards/rejected": -22.092905044555664, |
| "step": 2510 |
| }, |
| { |
| "epoch": 1.15, |
| "learning_rate": 2.054794520547945e-07, |
| "logits/chosen": -2.0834057331085205, |
| "logits/rejected": -1.7100751399993896, |
| "logps/chosen": -85.79044342041016, |
| "logps/rejected": -113.79996490478516, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7096903324127197, |
| "rewards/margins": 25.91143798828125, |
| "rewards/rejected": -23.20174789428711, |
| "step": 2520 |
| }, |
| { |
| "epoch": 1.15, |
| "learning_rate": 2.0497209538305426e-07, |
| "logits/chosen": -2.2217605113983154, |
| "logits/rejected": -1.871063470840454, |
| "logps/chosen": -87.78010559082031, |
| "logps/rejected": -115.4132080078125, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.101217746734619, |
| "rewards/margins": 25.03754234313965, |
| "rewards/rejected": -22.93632698059082, |
| "step": 2530 |
| }, |
| { |
| "epoch": 1.16, |
| "learning_rate": 2.0446473871131402e-07, |
| "logits/chosen": -2.188868761062622, |
| "logits/rejected": -1.7949635982513428, |
| "logps/chosen": -90.8096923828125, |
| "logps/rejected": -122.7888412475586, |
| "loss": 0.0017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.599236488342285, |
| "rewards/margins": 25.132827758789062, |
| "rewards/rejected": -22.533588409423828, |
| "step": 2540 |
| }, |
| { |
| "epoch": 1.16, |
| "learning_rate": 2.039573820395738e-07, |
| "logits/chosen": -2.1523804664611816, |
| "logits/rejected": -1.8026365041732788, |
| "logps/chosen": -79.64842224121094, |
| "logps/rejected": -115.60630798339844, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1584062576293945, |
| "rewards/margins": 25.834789276123047, |
| "rewards/rejected": -23.67638397216797, |
| "step": 2550 |
| }, |
| { |
| "epoch": 1.17, |
| "learning_rate": 2.0345002536783356e-07, |
| "logits/chosen": -2.191920757293701, |
| "logits/rejected": -1.7900508642196655, |
| "logps/chosen": -81.1897964477539, |
| "logps/rejected": -110.9433822631836, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1984918117523193, |
| "rewards/margins": 25.696517944335938, |
| "rewards/rejected": -22.498027801513672, |
| "step": 2560 |
| }, |
| { |
| "epoch": 1.17, |
| "learning_rate": 2.0294266869609332e-07, |
| "logits/chosen": -2.240447998046875, |
| "logits/rejected": -1.842795729637146, |
| "logps/chosen": -89.96078491210938, |
| "logps/rejected": -115.45732116699219, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5964362621307373, |
| "rewards/margins": 26.703838348388672, |
| "rewards/rejected": -24.107402801513672, |
| "step": 2570 |
| }, |
| { |
| "epoch": 1.18, |
| "learning_rate": 2.024353120243531e-07, |
| "logits/chosen": -2.155785083770752, |
| "logits/rejected": -1.7801926136016846, |
| "logps/chosen": -79.60318756103516, |
| "logps/rejected": -111.69859313964844, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1407876014709473, |
| "rewards/margins": 24.803207397460938, |
| "rewards/rejected": -21.66242027282715, |
| "step": 2580 |
| }, |
| { |
| "epoch": 1.18, |
| "learning_rate": 2.0192795535261286e-07, |
| "logits/chosen": -2.186657428741455, |
| "logits/rejected": -1.8247960805892944, |
| "logps/chosen": -83.01274108886719, |
| "logps/rejected": -115.82133483886719, |
| "loss": 0.0055, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.050610065460205, |
| "rewards/margins": 26.237497329711914, |
| "rewards/rejected": -23.1868839263916, |
| "step": 2590 |
| }, |
| { |
| "epoch": 1.19, |
| "learning_rate": 2.0142059868087262e-07, |
| "logits/chosen": -2.184058666229248, |
| "logits/rejected": -1.8208153247833252, |
| "logps/chosen": -87.23689270019531, |
| "logps/rejected": -112.4271240234375, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2483184337615967, |
| "rewards/margins": 25.097911834716797, |
| "rewards/rejected": -21.849592208862305, |
| "step": 2600 |
| }, |
| { |
| "epoch": 1.19, |
| "eval_logits/chosen": -2.1579010486602783, |
| "eval_logits/rejected": -1.8120908737182617, |
| "eval_logps/chosen": -83.75383758544922, |
| "eval_logps/rejected": -112.24433898925781, |
| "eval_loss": 0.005189881194382906, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.5383784770965576, |
| "eval_rewards/margins": 25.245080947875977, |
| "eval_rewards/rejected": -22.706703186035156, |
| "eval_runtime": 299.5228, |
| "eval_samples_per_second": 9.555, |
| "eval_steps_per_second": 0.598, |
| "step": 2600 |
| }, |
| { |
| "epoch": 1.19, |
| "learning_rate": 2.009132420091324e-07, |
| "logits/chosen": -2.0997519493103027, |
| "logits/rejected": -1.7868757247924805, |
| "logps/chosen": -81.20166015625, |
| "logps/rejected": -114.8741455078125, |
| "loss": 0.0065, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8495497703552246, |
| "rewards/margins": 26.784374237060547, |
| "rewards/rejected": -23.934823989868164, |
| "step": 2610 |
| }, |
| { |
| "epoch": 1.2, |
| "learning_rate": 2.0040588533739216e-07, |
| "logits/chosen": -2.1891965866088867, |
| "logits/rejected": -1.8037872314453125, |
| "logps/chosen": -86.70457458496094, |
| "logps/rejected": -116.2646255493164, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.284919261932373, |
| "rewards/margins": 26.434612274169922, |
| "rewards/rejected": -24.149694442749023, |
| "step": 2620 |
| }, |
| { |
| "epoch": 1.2, |
| "learning_rate": 1.9989852866565192e-07, |
| "logits/chosen": -2.1447997093200684, |
| "logits/rejected": -1.8104356527328491, |
| "logps/chosen": -85.48689270019531, |
| "logps/rejected": -120.234375, |
| "loss": 0.0048, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.3036551475524902, |
| "rewards/margins": 25.647720336914062, |
| "rewards/rejected": -24.344066619873047, |
| "step": 2630 |
| }, |
| { |
| "epoch": 1.21, |
| "learning_rate": 1.993911719939117e-07, |
| "logits/chosen": -2.1070830821990967, |
| "logits/rejected": -1.7089662551879883, |
| "logps/chosen": -88.43952178955078, |
| "logps/rejected": -112.84476470947266, |
| "loss": 0.0039, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.554791212081909, |
| "rewards/margins": 25.463603973388672, |
| "rewards/rejected": -22.9088134765625, |
| "step": 2640 |
| }, |
| { |
| "epoch": 1.21, |
| "learning_rate": 1.9888381532217146e-07, |
| "logits/chosen": -2.1640028953552246, |
| "logits/rejected": -1.803934097290039, |
| "logps/chosen": -87.4181900024414, |
| "logps/rejected": -119.70159912109375, |
| "loss": 0.0037, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3634817600250244, |
| "rewards/margins": 25.766544342041016, |
| "rewards/rejected": -24.403064727783203, |
| "step": 2650 |
| }, |
| { |
| "epoch": 1.21, |
| "learning_rate": 1.9837645865043122e-07, |
| "logits/chosen": -2.202446460723877, |
| "logits/rejected": -1.8282486200332642, |
| "logps/chosen": -87.19379425048828, |
| "logps/rejected": -118.42472839355469, |
| "loss": 0.0024, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8503637313842773, |
| "rewards/margins": 26.071752548217773, |
| "rewards/rejected": -24.22138786315918, |
| "step": 2660 |
| }, |
| { |
| "epoch": 1.22, |
| "learning_rate": 1.97869101978691e-07, |
| "logits/chosen": -2.149972915649414, |
| "logits/rejected": -1.789910912513733, |
| "logps/chosen": -87.65351867675781, |
| "logps/rejected": -117.45466613769531, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0328145027160645, |
| "rewards/margins": 25.302623748779297, |
| "rewards/rejected": -23.269811630249023, |
| "step": 2670 |
| }, |
| { |
| "epoch": 1.22, |
| "learning_rate": 1.9736174530695076e-07, |
| "logits/chosen": -2.2164740562438965, |
| "logits/rejected": -1.8319326639175415, |
| "logps/chosen": -87.3599853515625, |
| "logps/rejected": -120.3600845336914, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.874768853187561, |
| "rewards/margins": 27.10894775390625, |
| "rewards/rejected": -25.234180450439453, |
| "step": 2680 |
| }, |
| { |
| "epoch": 1.23, |
| "learning_rate": 1.9685438863521052e-07, |
| "logits/chosen": -2.1731603145599365, |
| "logits/rejected": -1.781818151473999, |
| "logps/chosen": -86.36930847167969, |
| "logps/rejected": -117.47142028808594, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4858036041259766, |
| "rewards/margins": 26.315841674804688, |
| "rewards/rejected": -23.83003807067871, |
| "step": 2690 |
| }, |
| { |
| "epoch": 1.23, |
| "learning_rate": 1.963470319634703e-07, |
| "logits/chosen": -2.2734062671661377, |
| "logits/rejected": -1.908062219619751, |
| "logps/chosen": -84.78250122070312, |
| "logps/rejected": -117.51930236816406, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3912415504455566, |
| "rewards/margins": 27.765466690063477, |
| "rewards/rejected": -25.37422752380371, |
| "step": 2700 |
| }, |
| { |
| "epoch": 1.23, |
| "eval_logits/chosen": -2.178615093231201, |
| "eval_logits/rejected": -1.8322229385375977, |
| "eval_logps/chosen": -86.60983276367188, |
| "eval_logps/rejected": -116.65242767333984, |
| "eval_loss": 0.005201002117246389, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.1103774309158325, |
| "eval_rewards/margins": 26.021133422851562, |
| "eval_rewards/rejected": -24.910757064819336, |
| "eval_runtime": 204.9916, |
| "eval_samples_per_second": 13.962, |
| "eval_steps_per_second": 0.873, |
| "step": 2700 |
| }, |
| { |
| "epoch": 1.24, |
| "learning_rate": 1.9583967529173006e-07, |
| "logits/chosen": -2.1784110069274902, |
| "logits/rejected": -1.8004591464996338, |
| "logps/chosen": -90.504638671875, |
| "logps/rejected": -120.81787109375, |
| "loss": 0.0014, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6989917755126953, |
| "rewards/margins": 27.467670440673828, |
| "rewards/rejected": -25.7686767578125, |
| "step": 2710 |
| }, |
| { |
| "epoch": 1.24, |
| "learning_rate": 1.9533231861998982e-07, |
| "logits/chosen": -2.2853636741638184, |
| "logits/rejected": -1.9385350942611694, |
| "logps/chosen": -88.9593734741211, |
| "logps/rejected": -126.08199310302734, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.512266993522644, |
| "rewards/margins": 27.1735782623291, |
| "rewards/rejected": -25.661312103271484, |
| "step": 2720 |
| }, |
| { |
| "epoch": 1.25, |
| "learning_rate": 1.948249619482496e-07, |
| "logits/chosen": -2.0956151485443115, |
| "logits/rejected": -1.7385085821151733, |
| "logps/chosen": -88.5088882446289, |
| "logps/rejected": -119.0416488647461, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1305296421051025, |
| "rewards/margins": 26.505001068115234, |
| "rewards/rejected": -25.374475479125977, |
| "step": 2730 |
| }, |
| { |
| "epoch": 1.25, |
| "learning_rate": 1.9431760527650936e-07, |
| "logits/chosen": -2.2182435989379883, |
| "logits/rejected": -1.8027465343475342, |
| "logps/chosen": -87.2740707397461, |
| "logps/rejected": -126.13343811035156, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.500243663787842, |
| "rewards/margins": 29.380783081054688, |
| "rewards/rejected": -26.880542755126953, |
| "step": 2740 |
| }, |
| { |
| "epoch": 1.26, |
| "learning_rate": 1.9381024860476912e-07, |
| "logits/chosen": -2.2416915893554688, |
| "logits/rejected": -1.851488471031189, |
| "logps/chosen": -92.11241149902344, |
| "logps/rejected": -120.4324722290039, |
| "loss": 0.0055, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0522854328155518, |
| "rewards/margins": 27.805065155029297, |
| "rewards/rejected": -25.75278091430664, |
| "step": 2750 |
| }, |
| { |
| "epoch": 1.26, |
| "learning_rate": 1.933028919330289e-07, |
| "logits/chosen": -2.1997532844543457, |
| "logits/rejected": -1.8702919483184814, |
| "logps/chosen": -87.5173568725586, |
| "logps/rejected": -121.913818359375, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8851993680000305, |
| "rewards/margins": 27.617206573486328, |
| "rewards/rejected": -26.732006072998047, |
| "step": 2760 |
| }, |
| { |
| "epoch": 1.26, |
| "learning_rate": 1.9279553526128866e-07, |
| "logits/chosen": -2.212709426879883, |
| "logits/rejected": -1.8357082605361938, |
| "logps/chosen": -85.00953674316406, |
| "logps/rejected": -122.2784652709961, |
| "loss": 0.0069, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2315926551818848, |
| "rewards/margins": 29.97836685180664, |
| "rewards/rejected": -27.746774673461914, |
| "step": 2770 |
| }, |
| { |
| "epoch": 1.27, |
| "learning_rate": 1.9228817858954842e-07, |
| "logits/chosen": -2.2300631999969482, |
| "logits/rejected": -1.828784704208374, |
| "logps/chosen": -84.7209243774414, |
| "logps/rejected": -122.51011657714844, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.3681228160858154, |
| "rewards/margins": 29.574649810791016, |
| "rewards/rejected": -26.206527709960938, |
| "step": 2780 |
| }, |
| { |
| "epoch": 1.27, |
| "learning_rate": 1.917808219178082e-07, |
| "logits/chosen": -2.255375385284424, |
| "logits/rejected": -1.8779224157333374, |
| "logps/chosen": -85.09639739990234, |
| "logps/rejected": -115.9112548828125, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2712059020996094, |
| "rewards/margins": 27.021814346313477, |
| "rewards/rejected": -23.750606536865234, |
| "step": 2790 |
| }, |
| { |
| "epoch": 1.28, |
| "learning_rate": 1.9127346524606796e-07, |
| "logits/chosen": -2.224299430847168, |
| "logits/rejected": -1.8939344882965088, |
| "logps/chosen": -82.23179626464844, |
| "logps/rejected": -113.6113052368164, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4387083053588867, |
| "rewards/margins": 26.06760025024414, |
| "rewards/rejected": -23.62889289855957, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.28, |
| "eval_logits/chosen": -2.1937084197998047, |
| "eval_logits/rejected": -1.8446825742721558, |
| "eval_logps/chosen": -84.96446990966797, |
| "eval_logps/rejected": -116.5991439819336, |
| "eval_loss": 0.005613674875348806, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.9330565929412842, |
| "eval_rewards/margins": 26.81716537475586, |
| "eval_rewards/rejected": -24.88410758972168, |
| "eval_runtime": 259.8625, |
| "eval_samples_per_second": 11.014, |
| "eval_steps_per_second": 0.689, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.28, |
| "learning_rate": 1.9076610857432772e-07, |
| "logits/chosen": -2.2159202098846436, |
| "logits/rejected": -1.7732995748519897, |
| "logps/chosen": -86.9103775024414, |
| "logps/rejected": -119.67277526855469, |
| "loss": 0.001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8791584968566895, |
| "rewards/margins": 28.936452865600586, |
| "rewards/rejected": -26.05729103088379, |
| "step": 2810 |
| }, |
| { |
| "epoch": 1.29, |
| "learning_rate": 1.902587519025875e-07, |
| "logits/chosen": -2.169167995452881, |
| "logits/rejected": -1.750314474105835, |
| "logps/chosen": -89.01744079589844, |
| "logps/rejected": -120.8212661743164, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.396334171295166, |
| "rewards/margins": 29.005077362060547, |
| "rewards/rejected": -26.60874366760254, |
| "step": 2820 |
| }, |
| { |
| "epoch": 1.29, |
| "learning_rate": 1.8975139523084726e-07, |
| "logits/chosen": -2.1312053203582764, |
| "logits/rejected": -1.7972911596298218, |
| "logps/chosen": -86.7480697631836, |
| "logps/rejected": -125.67042541503906, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.189509391784668, |
| "rewards/margins": 28.36881446838379, |
| "rewards/rejected": -26.179306030273438, |
| "step": 2830 |
| }, |
| { |
| "epoch": 1.3, |
| "learning_rate": 1.8924403855910702e-07, |
| "logits/chosen": -2.2344472408294678, |
| "logits/rejected": -1.9485044479370117, |
| "logps/chosen": -79.72859954833984, |
| "logps/rejected": -119.08320617675781, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4703823328018188, |
| "rewards/margins": 27.174612045288086, |
| "rewards/rejected": -25.7042293548584, |
| "step": 2840 |
| }, |
| { |
| "epoch": 1.3, |
| "learning_rate": 1.887366818873668e-07, |
| "logits/chosen": -2.237384080886841, |
| "logits/rejected": -1.8804075717926025, |
| "logps/chosen": -87.28046417236328, |
| "logps/rejected": -117.6243896484375, |
| "loss": 0.0003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6116135120391846, |
| "rewards/margins": 26.46062660217285, |
| "rewards/rejected": -24.849010467529297, |
| "step": 2850 |
| }, |
| { |
| "epoch": 1.31, |
| "learning_rate": 1.8822932521562656e-07, |
| "logits/chosen": -2.1050355434417725, |
| "logits/rejected": -1.7900664806365967, |
| "logps/chosen": -87.86946105957031, |
| "logps/rejected": -118.79146575927734, |
| "loss": 0.0076, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.9105596542358398, |
| "rewards/margins": 25.8955020904541, |
| "rewards/rejected": -23.984943389892578, |
| "step": 2860 |
| }, |
| { |
| "epoch": 1.31, |
| "learning_rate": 1.8772196854388632e-07, |
| "logits/chosen": -2.2055516242980957, |
| "logits/rejected": -1.794002890586853, |
| "logps/chosen": -90.33226013183594, |
| "logps/rejected": -122.12618255615234, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3315025568008423, |
| "rewards/margins": 27.155811309814453, |
| "rewards/rejected": -25.82430648803711, |
| "step": 2870 |
| }, |
| { |
| "epoch": 1.31, |
| "learning_rate": 1.872146118721461e-07, |
| "logits/chosen": -2.1890950202941895, |
| "logits/rejected": -1.7347943782806396, |
| "logps/chosen": -93.88822937011719, |
| "logps/rejected": -118.97929382324219, |
| "loss": 0.0011, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.2069919109344482, |
| "rewards/margins": 27.36787986755371, |
| "rewards/rejected": -25.160892486572266, |
| "step": 2880 |
| }, |
| { |
| "epoch": 1.32, |
| "learning_rate": 1.8670725520040586e-07, |
| "logits/chosen": -2.2413697242736816, |
| "logits/rejected": -1.8580372333526611, |
| "logps/chosen": -88.85777282714844, |
| "logps/rejected": -122.02364349365234, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6634600162506104, |
| "rewards/margins": 29.12453842163086, |
| "rewards/rejected": -26.46108055114746, |
| "step": 2890 |
| }, |
| { |
| "epoch": 1.32, |
| "learning_rate": 1.8619989852866562e-07, |
| "logits/chosen": -2.1835293769836426, |
| "logits/rejected": -1.8364540338516235, |
| "logps/chosen": -87.46830749511719, |
| "logps/rejected": -118.46342468261719, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9554582834243774, |
| "rewards/margins": 27.929412841796875, |
| "rewards/rejected": -25.973957061767578, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.32, |
| "eval_logits/chosen": -2.1951773166656494, |
| "eval_logits/rejected": -1.8438202142715454, |
| "eval_logps/chosen": -85.55497741699219, |
| "eval_logps/rejected": -119.35179901123047, |
| "eval_loss": 0.005554942414164543, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.6378037929534912, |
| "eval_rewards/margins": 27.898239135742188, |
| "eval_rewards/rejected": -26.26043701171875, |
| "eval_runtime": 327.6474, |
| "eval_samples_per_second": 8.735, |
| "eval_steps_per_second": 0.546, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.33, |
| "learning_rate": 1.856925418569254e-07, |
| "logits/chosen": -2.1702980995178223, |
| "logits/rejected": -1.7541742324829102, |
| "logps/chosen": -87.9078598022461, |
| "logps/rejected": -117.9532699584961, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.8803863525390625, |
| "rewards/margins": 28.9564151763916, |
| "rewards/rejected": -25.07602882385254, |
| "step": 2910 |
| }, |
| { |
| "epoch": 1.33, |
| "learning_rate": 1.8518518518518516e-07, |
| "logits/chosen": -2.164057493209839, |
| "logits/rejected": -1.8279097080230713, |
| "logps/chosen": -82.57586669921875, |
| "logps/rejected": -120.59794616699219, |
| "loss": 0.0053, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.3020541667938232, |
| "rewards/margins": 29.443384170532227, |
| "rewards/rejected": -27.141326904296875, |
| "step": 2920 |
| }, |
| { |
| "epoch": 1.34, |
| "learning_rate": 1.8467782851344492e-07, |
| "logits/chosen": -2.170772075653076, |
| "logits/rejected": -1.8161497116088867, |
| "logps/chosen": -85.89886474609375, |
| "logps/rejected": -123.47340393066406, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1593396663665771, |
| "rewards/margins": 29.278268814086914, |
| "rewards/rejected": -28.118927001953125, |
| "step": 2930 |
| }, |
| { |
| "epoch": 1.34, |
| "learning_rate": 1.841704718417047e-07, |
| "logits/chosen": -2.198502779006958, |
| "logits/rejected": -1.8595365285873413, |
| "logps/chosen": -82.33607482910156, |
| "logps/rejected": -117.26557922363281, |
| "loss": 0.0058, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3132636547088623, |
| "rewards/margins": 27.13201904296875, |
| "rewards/rejected": -25.818756103515625, |
| "step": 2940 |
| }, |
| { |
| "epoch": 1.35, |
| "learning_rate": 1.8366311516996446e-07, |
| "logits/chosen": -2.2103936672210693, |
| "logits/rejected": -1.8283309936523438, |
| "logps/chosen": -87.3060302734375, |
| "logps/rejected": -116.38105773925781, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.252316951751709, |
| "rewards/margins": 27.21462059020996, |
| "rewards/rejected": -24.962305068969727, |
| "step": 2950 |
| }, |
| { |
| "epoch": 1.35, |
| "learning_rate": 1.8315575849822422e-07, |
| "logits/chosen": -2.1284189224243164, |
| "logits/rejected": -1.7991136312484741, |
| "logps/chosen": -85.66838073730469, |
| "logps/rejected": -115.23576354980469, |
| "loss": 0.0017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.003413677215576, |
| "rewards/margins": 25.363988876342773, |
| "rewards/rejected": -23.360576629638672, |
| "step": 2960 |
| }, |
| { |
| "epoch": 1.36, |
| "learning_rate": 1.82648401826484e-07, |
| "logits/chosen": -2.1712751388549805, |
| "logits/rejected": -1.8838971853256226, |
| "logps/chosen": -81.59526824951172, |
| "logps/rejected": -119.744384765625, |
| "loss": 0.0049, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3410313129425049, |
| "rewards/margins": 26.60662269592285, |
| "rewards/rejected": -25.265588760375977, |
| "step": 2970 |
| }, |
| { |
| "epoch": 1.36, |
| "learning_rate": 1.8214104515474375e-07, |
| "logits/chosen": -2.229074239730835, |
| "logits/rejected": -1.8246219158172607, |
| "logps/chosen": -85.791259765625, |
| "logps/rejected": -122.32413482666016, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3884541988372803, |
| "rewards/margins": 29.200729370117188, |
| "rewards/rejected": -27.812274932861328, |
| "step": 2980 |
| }, |
| { |
| "epoch": 1.36, |
| "learning_rate": 1.8163368848300352e-07, |
| "logits/chosen": -2.2312495708465576, |
| "logits/rejected": -1.912697196006775, |
| "logps/chosen": -80.87403869628906, |
| "logps/rejected": -123.8989028930664, |
| "loss": 0.0083, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5714377164840698, |
| "rewards/margins": 28.29909324645996, |
| "rewards/rejected": -26.7276554107666, |
| "step": 2990 |
| }, |
| { |
| "epoch": 1.37, |
| "learning_rate": 1.811263318112633e-07, |
| "logits/chosen": -2.2286040782928467, |
| "logits/rejected": -1.8826186656951904, |
| "logps/chosen": -86.8649673461914, |
| "logps/rejected": -121.7653579711914, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1471002101898193, |
| "rewards/margins": 27.70977783203125, |
| "rewards/rejected": -25.562679290771484, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.37, |
| "eval_logits/chosen": -2.1800849437713623, |
| "eval_logits/rejected": -1.830121636390686, |
| "eval_logps/chosen": -86.1629867553711, |
| "eval_logps/rejected": -119.01561737060547, |
| "eval_loss": 0.006075535900890827, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.3337992429733276, |
| "eval_rewards/margins": 27.4261474609375, |
| "eval_rewards/rejected": -26.092342376708984, |
| "eval_runtime": 190.0895, |
| "eval_samples_per_second": 15.056, |
| "eval_steps_per_second": 0.942, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.37, |
| "learning_rate": 1.8061897513952305e-07, |
| "logits/chosen": -2.1978001594543457, |
| "logits/rejected": -1.7904773950576782, |
| "logps/chosen": -89.93728637695312, |
| "logps/rejected": -120.550048828125, |
| "loss": 0.0003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8481388092041016, |
| "rewards/margins": 28.82729721069336, |
| "rewards/rejected": -25.979156494140625, |
| "step": 3010 |
| }, |
| { |
| "epoch": 1.38, |
| "learning_rate": 1.8011161846778282e-07, |
| "logits/chosen": -2.20564603805542, |
| "logits/rejected": -1.8694099187850952, |
| "logps/chosen": -85.4388427734375, |
| "logps/rejected": -119.31624603271484, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.9251155853271484, |
| "rewards/margins": 26.277385711669922, |
| "rewards/rejected": -24.352270126342773, |
| "step": 3020 |
| }, |
| { |
| "epoch": 1.38, |
| "learning_rate": 1.796042617960426e-07, |
| "logits/chosen": -2.176328659057617, |
| "logits/rejected": -1.7874501943588257, |
| "logps/chosen": -97.2492446899414, |
| "logps/rejected": -123.11531066894531, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5614807605743408, |
| "rewards/margins": 26.78195571899414, |
| "rewards/rejected": -25.22047233581543, |
| "step": 3030 |
| }, |
| { |
| "epoch": 1.39, |
| "learning_rate": 1.7909690512430235e-07, |
| "logits/chosen": -2.1733341217041016, |
| "logits/rejected": -1.796979546546936, |
| "logps/chosen": -84.2280502319336, |
| "logps/rejected": -116.29603576660156, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3264310359954834, |
| "rewards/margins": 27.896953582763672, |
| "rewards/rejected": -25.57052230834961, |
| "step": 3040 |
| }, |
| { |
| "epoch": 1.39, |
| "learning_rate": 1.7858954845256212e-07, |
| "logits/chosen": -2.2226386070251465, |
| "logits/rejected": -1.8575359582901, |
| "logps/chosen": -85.23347473144531, |
| "logps/rejected": -116.77949523925781, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.571824550628662, |
| "rewards/margins": 27.728759765625, |
| "rewards/rejected": -25.156932830810547, |
| "step": 3050 |
| }, |
| { |
| "epoch": 1.4, |
| "learning_rate": 1.780821917808219e-07, |
| "logits/chosen": -2.196302652359009, |
| "logits/rejected": -1.8011735677719116, |
| "logps/chosen": -90.76152038574219, |
| "logps/rejected": -119.6912841796875, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5391037464141846, |
| "rewards/margins": 26.7655086517334, |
| "rewards/rejected": -24.226404190063477, |
| "step": 3060 |
| }, |
| { |
| "epoch": 1.4, |
| "learning_rate": 1.7757483510908165e-07, |
| "logits/chosen": -2.184532642364502, |
| "logits/rejected": -1.8580677509307861, |
| "logps/chosen": -84.7155990600586, |
| "logps/rejected": -126.76289367675781, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2373816967010498, |
| "rewards/margins": 28.667322158813477, |
| "rewards/rejected": -27.4299373626709, |
| "step": 3070 |
| }, |
| { |
| "epoch": 1.41, |
| "learning_rate": 1.7706747843734142e-07, |
| "logits/chosen": -2.236210823059082, |
| "logits/rejected": -1.8297055959701538, |
| "logps/chosen": -84.67916107177734, |
| "logps/rejected": -119.5815200805664, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.601902723312378, |
| "rewards/margins": 28.8743896484375, |
| "rewards/rejected": -27.27248764038086, |
| "step": 3080 |
| }, |
| { |
| "epoch": 1.41, |
| "learning_rate": 1.765601217656012e-07, |
| "logits/chosen": -2.2311558723449707, |
| "logits/rejected": -1.878273367881775, |
| "logps/chosen": -88.32709503173828, |
| "logps/rejected": -121.57609558105469, |
| "loss": 0.0054, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2206145524978638, |
| "rewards/margins": 28.100128173828125, |
| "rewards/rejected": -26.879512786865234, |
| "step": 3090 |
| }, |
| { |
| "epoch": 1.42, |
| "learning_rate": 1.7605276509386095e-07, |
| "logits/chosen": -2.1835224628448486, |
| "logits/rejected": -1.8489364385604858, |
| "logps/chosen": -84.09500122070312, |
| "logps/rejected": -121.5748519897461, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4824763238430023, |
| "rewards/margins": 27.168865203857422, |
| "rewards/rejected": -26.686386108398438, |
| "step": 3100 |
| }, |
| { |
| "epoch": 1.42, |
| "eval_logits/chosen": -2.176138162612915, |
| "eval_logits/rejected": -1.8300259113311768, |
| "eval_logps/chosen": -86.61859130859375, |
| "eval_logps/rejected": -119.87804412841797, |
| "eval_loss": 0.0059745111502707005, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.1059939861297607, |
| "eval_rewards/margins": 27.629554748535156, |
| "eval_rewards/rejected": -26.523563385009766, |
| "eval_runtime": 207.1978, |
| "eval_samples_per_second": 13.813, |
| "eval_steps_per_second": 0.864, |
| "step": 3100 |
| }, |
| { |
| "epoch": 1.42, |
| "learning_rate": 1.7554540842212072e-07, |
| "logits/chosen": -2.169111728668213, |
| "logits/rejected": -1.8405656814575195, |
| "logps/chosen": -83.15672302246094, |
| "logps/rejected": -120.4351577758789, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0360536575317383, |
| "rewards/margins": 27.2529354095459, |
| "rewards/rejected": -26.216882705688477, |
| "step": 3110 |
| }, |
| { |
| "epoch": 1.42, |
| "learning_rate": 1.750380517503805e-07, |
| "logits/chosen": -2.2091641426086426, |
| "logits/rejected": -1.8550224304199219, |
| "logps/chosen": -84.96271514892578, |
| "logps/rejected": -119.28694152832031, |
| "loss": 0.0052, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8836094737052917, |
| "rewards/margins": 26.456172943115234, |
| "rewards/rejected": -25.572561264038086, |
| "step": 3120 |
| }, |
| { |
| "epoch": 1.43, |
| "learning_rate": 1.7453069507864025e-07, |
| "logits/chosen": -2.2117037773132324, |
| "logits/rejected": -1.8669350147247314, |
| "logps/chosen": -84.34877014160156, |
| "logps/rejected": -119.05101013183594, |
| "loss": 0.0076, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.0027484893798828, |
| "rewards/margins": 28.111114501953125, |
| "rewards/rejected": -27.10836410522461, |
| "step": 3130 |
| }, |
| { |
| "epoch": 1.43, |
| "learning_rate": 1.7402333840690002e-07, |
| "logits/chosen": -2.1435580253601074, |
| "logits/rejected": -1.753458023071289, |
| "logps/chosen": -93.03610229492188, |
| "logps/rejected": -126.67008972167969, |
| "loss": 0.0008, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7019534707069397, |
| "rewards/margins": 28.131439208984375, |
| "rewards/rejected": -27.42948341369629, |
| "step": 3140 |
| }, |
| { |
| "epoch": 1.44, |
| "learning_rate": 1.735159817351598e-07, |
| "logits/chosen": -2.1424784660339355, |
| "logits/rejected": -1.8056213855743408, |
| "logps/chosen": -82.12688446044922, |
| "logps/rejected": -125.8278579711914, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.45741605758667, |
| "rewards/margins": 30.075420379638672, |
| "rewards/rejected": -27.61800765991211, |
| "step": 3150 |
| }, |
| { |
| "epoch": 1.44, |
| "learning_rate": 1.7300862506341955e-07, |
| "logits/chosen": -2.2541353702545166, |
| "logits/rejected": -1.8698198795318604, |
| "logps/chosen": -87.37802124023438, |
| "logps/rejected": -123.66682434082031, |
| "loss": 0.0064, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.516167402267456, |
| "rewards/margins": 29.741958618164062, |
| "rewards/rejected": -28.22579002380371, |
| "step": 3160 |
| }, |
| { |
| "epoch": 1.45, |
| "learning_rate": 1.7250126839167932e-07, |
| "logits/chosen": -2.297926902770996, |
| "logits/rejected": -1.9295371770858765, |
| "logps/chosen": -88.74690246582031, |
| "logps/rejected": -119.91023254394531, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9074515104293823, |
| "rewards/margins": 27.017669677734375, |
| "rewards/rejected": -26.110218048095703, |
| "step": 3170 |
| }, |
| { |
| "epoch": 1.45, |
| "learning_rate": 1.719939117199391e-07, |
| "logits/chosen": -2.1698784828186035, |
| "logits/rejected": -1.8197612762451172, |
| "logps/chosen": -87.33271789550781, |
| "logps/rejected": -120.39599609375, |
| "loss": 0.0037, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6318087577819824, |
| "rewards/margins": 29.638574600219727, |
| "rewards/rejected": -27.006765365600586, |
| "step": 3180 |
| }, |
| { |
| "epoch": 1.46, |
| "learning_rate": 1.7148655504819885e-07, |
| "logits/chosen": -2.126783847808838, |
| "logits/rejected": -1.7983070611953735, |
| "logps/chosen": -83.7030029296875, |
| "logps/rejected": -122.6335678100586, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.919542670249939, |
| "rewards/margins": 28.071773529052734, |
| "rewards/rejected": -26.152231216430664, |
| "step": 3190 |
| }, |
| { |
| "epoch": 1.46, |
| "learning_rate": 1.7097919837645862e-07, |
| "logits/chosen": -2.227081298828125, |
| "logits/rejected": -1.9020026922225952, |
| "logps/chosen": -85.4066162109375, |
| "logps/rejected": -123.98991394042969, |
| "loss": 0.0031, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.6433346271514893, |
| "rewards/margins": 29.48199462890625, |
| "rewards/rejected": -27.838659286499023, |
| "step": 3200 |
| }, |
| { |
| "epoch": 1.46, |
| "eval_logits/chosen": -2.1810193061828613, |
| "eval_logits/rejected": -1.8324401378631592, |
| "eval_logps/chosen": -85.56733703613281, |
| "eval_logps/rejected": -119.99109649658203, |
| "eval_loss": 0.00612166291102767, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.631625771522522, |
| "eval_rewards/margins": 28.211711883544922, |
| "eval_rewards/rejected": -26.58008575439453, |
| "eval_runtime": 214.0667, |
| "eval_samples_per_second": 13.37, |
| "eval_steps_per_second": 0.836, |
| "step": 3200 |
| }, |
| { |
| "epoch": 1.47, |
| "learning_rate": 1.704718417047184e-07, |
| "logits/chosen": -2.2383410930633545, |
| "logits/rejected": -1.9348970651626587, |
| "logps/chosen": -86.20955657958984, |
| "logps/rejected": -122.5006103515625, |
| "loss": 0.0029, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4148099422454834, |
| "rewards/margins": 27.235687255859375, |
| "rewards/rejected": -24.82087516784668, |
| "step": 3210 |
| }, |
| { |
| "epoch": 1.47, |
| "learning_rate": 1.6996448503297815e-07, |
| "logits/chosen": -2.160952091217041, |
| "logits/rejected": -1.7654712200164795, |
| "logps/chosen": -89.77068328857422, |
| "logps/rejected": -118.8796615600586, |
| "loss": 0.004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.054098606109619, |
| "rewards/margins": 26.069538116455078, |
| "rewards/rejected": -24.01543617248535, |
| "step": 3220 |
| }, |
| { |
| "epoch": 1.47, |
| "learning_rate": 1.6945712836123792e-07, |
| "logits/chosen": -2.2201457023620605, |
| "logits/rejected": -1.820336937904358, |
| "logps/chosen": -84.93563842773438, |
| "logps/rejected": -115.88444519042969, |
| "loss": 0.0066, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6848886013031006, |
| "rewards/margins": 26.666845321655273, |
| "rewards/rejected": -23.98195457458496, |
| "step": 3230 |
| }, |
| { |
| "epoch": 1.48, |
| "learning_rate": 1.689497716894977e-07, |
| "logits/chosen": -2.1646978855133057, |
| "logits/rejected": -1.8357467651367188, |
| "logps/chosen": -80.21171569824219, |
| "logps/rejected": -111.47000885009766, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.577789306640625, |
| "rewards/margins": 25.826122283935547, |
| "rewards/rejected": -23.248332977294922, |
| "step": 3240 |
| }, |
| { |
| "epoch": 1.48, |
| "learning_rate": 1.6844241501775745e-07, |
| "logits/chosen": -2.1942062377929688, |
| "logits/rejected": -1.831578254699707, |
| "logps/chosen": -87.41214752197266, |
| "logps/rejected": -118.68165588378906, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.257878065109253, |
| "rewards/margins": 27.593063354492188, |
| "rewards/rejected": -24.335186004638672, |
| "step": 3250 |
| }, |
| { |
| "epoch": 1.49, |
| "learning_rate": 1.6793505834601722e-07, |
| "logits/chosen": -2.270230293273926, |
| "logits/rejected": -1.930605173110962, |
| "logps/chosen": -77.88417053222656, |
| "logps/rejected": -119.29927062988281, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.550905704498291, |
| "rewards/margins": 28.38033103942871, |
| "rewards/rejected": -25.82942771911621, |
| "step": 3260 |
| }, |
| { |
| "epoch": 1.49, |
| "learning_rate": 1.67427701674277e-07, |
| "logits/chosen": -2.1990127563476562, |
| "logits/rejected": -1.842660665512085, |
| "logps/chosen": -82.07890319824219, |
| "logps/rejected": -117.83439636230469, |
| "loss": 0.0028, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.725590467453003, |
| "rewards/margins": 28.352636337280273, |
| "rewards/rejected": -25.627044677734375, |
| "step": 3270 |
| }, |
| { |
| "epoch": 1.5, |
| "learning_rate": 1.6692034500253675e-07, |
| "logits/chosen": -2.181185245513916, |
| "logits/rejected": -1.8166754245758057, |
| "logps/chosen": -82.80810546875, |
| "logps/rejected": -124.4463882446289, |
| "loss": 0.0037, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.80354642868042, |
| "rewards/margins": 29.090805053710938, |
| "rewards/rejected": -26.287261962890625, |
| "step": 3280 |
| }, |
| { |
| "epoch": 1.5, |
| "learning_rate": 1.6641298833079652e-07, |
| "logits/chosen": -2.245620012283325, |
| "logits/rejected": -1.8439216613769531, |
| "logps/chosen": -87.42176055908203, |
| "logps/rejected": -118.27108001708984, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.304450273513794, |
| "rewards/margins": 28.731517791748047, |
| "rewards/rejected": -25.42706871032715, |
| "step": 3290 |
| }, |
| { |
| "epoch": 1.51, |
| "learning_rate": 1.659056316590563e-07, |
| "logits/chosen": -2.1820719242095947, |
| "logits/rejected": -1.8486725091934204, |
| "logps/chosen": -83.21141815185547, |
| "logps/rejected": -117.5422592163086, |
| "loss": 0.0018, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.1327810287475586, |
| "rewards/margins": 28.282459259033203, |
| "rewards/rejected": -26.149677276611328, |
| "step": 3300 |
| }, |
| { |
| "epoch": 1.51, |
| "eval_logits/chosen": -2.188385486602783, |
| "eval_logits/rejected": -1.8376048803329468, |
| "eval_logps/chosen": -84.18167114257812, |
| "eval_logps/rejected": -117.2090072631836, |
| "eval_loss": 0.005902700126171112, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 2.3244550228118896, |
| "eval_rewards/margins": 27.51349639892578, |
| "eval_rewards/rejected": -25.189043045043945, |
| "eval_runtime": 191.0473, |
| "eval_samples_per_second": 14.981, |
| "eval_steps_per_second": 0.937, |
| "step": 3300 |
| }, |
| { |
| "epoch": 1.51, |
| "learning_rate": 1.6539827498731605e-07, |
| "logits/chosen": -2.2313265800476074, |
| "logits/rejected": -1.8483736515045166, |
| "logps/chosen": -87.50598907470703, |
| "logps/rejected": -118.2992935180664, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3828155994415283, |
| "rewards/margins": 27.8851261138916, |
| "rewards/rejected": -25.502309799194336, |
| "step": 3310 |
| }, |
| { |
| "epoch": 1.52, |
| "learning_rate": 1.6489091831557582e-07, |
| "logits/chosen": -2.143418312072754, |
| "logits/rejected": -1.8131214380264282, |
| "logps/chosen": -82.66302490234375, |
| "logps/rejected": -117.9818344116211, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9075496196746826, |
| "rewards/margins": 27.7532958984375, |
| "rewards/rejected": -24.845745086669922, |
| "step": 3320 |
| }, |
| { |
| "epoch": 1.52, |
| "learning_rate": 1.643835616438356e-07, |
| "logits/chosen": -2.185781955718994, |
| "logits/rejected": -1.8397849798202515, |
| "logps/chosen": -86.03587341308594, |
| "logps/rejected": -121.69071960449219, |
| "loss": 0.0037, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.5947184562683105, |
| "rewards/margins": 28.71170997619629, |
| "rewards/rejected": -26.116989135742188, |
| "step": 3330 |
| }, |
| { |
| "epoch": 1.52, |
| "learning_rate": 1.6387620497209535e-07, |
| "logits/chosen": -2.093822956085205, |
| "logits/rejected": -1.744879126548767, |
| "logps/chosen": -82.53218078613281, |
| "logps/rejected": -120.4306869506836, |
| "loss": 0.0069, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.362114429473877, |
| "rewards/margins": 28.930118560791016, |
| "rewards/rejected": -26.568002700805664, |
| "step": 3340 |
| }, |
| { |
| "epoch": 1.53, |
| "learning_rate": 1.6336884830035512e-07, |
| "logits/chosen": -2.222090244293213, |
| "logits/rejected": -1.889953851699829, |
| "logps/chosen": -84.85404968261719, |
| "logps/rejected": -122.89866638183594, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0494067668914795, |
| "rewards/margins": 27.54694175720215, |
| "rewards/rejected": -25.497535705566406, |
| "step": 3350 |
| }, |
| { |
| "epoch": 1.53, |
| "learning_rate": 1.6286149162861489e-07, |
| "logits/chosen": -2.2259693145751953, |
| "logits/rejected": -1.8098411560058594, |
| "logps/chosen": -87.61415100097656, |
| "logps/rejected": -129.11331176757812, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2813143730163574, |
| "rewards/margins": 30.5456485748291, |
| "rewards/rejected": -28.264331817626953, |
| "step": 3360 |
| }, |
| { |
| "epoch": 1.54, |
| "learning_rate": 1.6235413495687465e-07, |
| "logits/chosen": -2.115265369415283, |
| "logits/rejected": -1.7785238027572632, |
| "logps/chosen": -83.63951110839844, |
| "logps/rejected": -116.60148620605469, |
| "loss": 0.0029, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2996184825897217, |
| "rewards/margins": 27.973474502563477, |
| "rewards/rejected": -25.67385482788086, |
| "step": 3370 |
| }, |
| { |
| "epoch": 1.54, |
| "learning_rate": 1.6184677828513442e-07, |
| "logits/chosen": -2.213620185852051, |
| "logits/rejected": -1.853643774986267, |
| "logps/chosen": -84.76154327392578, |
| "logps/rejected": -120.73722839355469, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5600218772888184, |
| "rewards/margins": 28.535400390625, |
| "rewards/rejected": -25.97538185119629, |
| "step": 3380 |
| }, |
| { |
| "epoch": 1.55, |
| "learning_rate": 1.613394216133942e-07, |
| "logits/chosen": -2.2256524562835693, |
| "logits/rejected": -1.8734019994735718, |
| "logps/chosen": -82.6207046508789, |
| "logps/rejected": -123.33949279785156, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0278208255767822, |
| "rewards/margins": 29.47714614868164, |
| "rewards/rejected": -27.449321746826172, |
| "step": 3390 |
| }, |
| { |
| "epoch": 1.55, |
| "learning_rate": 1.6083206494165398e-07, |
| "logits/chosen": -2.247122049331665, |
| "logits/rejected": -1.9250261783599854, |
| "logps/chosen": -81.63235473632812, |
| "logps/rejected": -120.65080261230469, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.553596258163452, |
| "rewards/margins": 29.37432289123535, |
| "rewards/rejected": -26.820724487304688, |
| "step": 3400 |
| }, |
| { |
| "epoch": 1.55, |
| "eval_logits/chosen": -2.1875288486480713, |
| "eval_logits/rejected": -1.8437479734420776, |
| "eval_logps/chosen": -84.4741439819336, |
| "eval_logps/rejected": -119.88742065429688, |
| "eval_loss": 0.005948640406131744, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.1782193183898926, |
| "eval_rewards/margins": 28.70646858215332, |
| "eval_rewards/rejected": -26.528249740600586, |
| "eval_runtime": 170.1725, |
| "eval_samples_per_second": 16.818, |
| "eval_steps_per_second": 1.052, |
| "step": 3400 |
| }, |
| { |
| "epoch": 1.56, |
| "learning_rate": 1.6032470826991375e-07, |
| "logits/chosen": -2.18739914894104, |
| "logits/rejected": -1.8111976385116577, |
| "logps/chosen": -83.60111999511719, |
| "logps/rejected": -120.876220703125, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.492570400238037, |
| "rewards/margins": 29.177154541015625, |
| "rewards/rejected": -25.684585571289062, |
| "step": 3410 |
| }, |
| { |
| "epoch": 1.56, |
| "learning_rate": 1.598173515981735e-07, |
| "logits/chosen": -2.1494498252868652, |
| "logits/rejected": -1.715921401977539, |
| "logps/chosen": -87.95207214355469, |
| "logps/rejected": -117.0306396484375, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.5173392295837402, |
| "rewards/margins": 28.325458526611328, |
| "rewards/rejected": -24.80811882019043, |
| "step": 3420 |
| }, |
| { |
| "epoch": 1.57, |
| "learning_rate": 1.5930999492643328e-07, |
| "logits/chosen": -2.2305150032043457, |
| "logits/rejected": -1.8406226634979248, |
| "logps/chosen": -85.95257568359375, |
| "logps/rejected": -121.40510559082031, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6646008491516113, |
| "rewards/margins": 29.42586898803711, |
| "rewards/rejected": -26.76127052307129, |
| "step": 3430 |
| }, |
| { |
| "epoch": 1.57, |
| "learning_rate": 1.5880263825469305e-07, |
| "logits/chosen": -2.2470791339874268, |
| "logits/rejected": -1.903607726097107, |
| "logps/chosen": -83.02677917480469, |
| "logps/rejected": -118.3487777709961, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.975865125656128, |
| "rewards/margins": 28.316776275634766, |
| "rewards/rejected": -25.340911865234375, |
| "step": 3440 |
| }, |
| { |
| "epoch": 1.57, |
| "learning_rate": 1.582952815829528e-07, |
| "logits/chosen": -2.106412410736084, |
| "logits/rejected": -1.7845014333724976, |
| "logps/chosen": -85.77284240722656, |
| "logps/rejected": -122.04698181152344, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.0580241680145264, |
| "rewards/margins": 28.00246238708496, |
| "rewards/rejected": -24.94443702697754, |
| "step": 3450 |
| }, |
| { |
| "epoch": 1.58, |
| "learning_rate": 1.5778792491121258e-07, |
| "logits/chosen": -2.230118989944458, |
| "logits/rejected": -1.8575313091278076, |
| "logps/chosen": -84.6324462890625, |
| "logps/rejected": -119.6858901977539, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.278738498687744, |
| "rewards/margins": 30.146167755126953, |
| "rewards/rejected": -26.867427825927734, |
| "step": 3460 |
| }, |
| { |
| "epoch": 1.58, |
| "learning_rate": 1.5728056823947235e-07, |
| "logits/chosen": -2.245896577835083, |
| "logits/rejected": -1.8796007633209229, |
| "logps/chosen": -83.982666015625, |
| "logps/rejected": -119.01679992675781, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.464191436767578, |
| "rewards/margins": 29.175838470458984, |
| "rewards/rejected": -25.711650848388672, |
| "step": 3470 |
| }, |
| { |
| "epoch": 1.59, |
| "learning_rate": 1.567732115677321e-07, |
| "logits/chosen": -2.186904191970825, |
| "logits/rejected": -1.7904678583145142, |
| "logps/chosen": -88.25177001953125, |
| "logps/rejected": -118.5602798461914, |
| "loss": 0.0067, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 2.3008944988250732, |
| "rewards/margins": 27.154077529907227, |
| "rewards/rejected": -24.853181838989258, |
| "step": 3480 |
| }, |
| { |
| "epoch": 1.59, |
| "learning_rate": 1.5626585489599188e-07, |
| "logits/chosen": -2.129375457763672, |
| "logits/rejected": -1.7635730504989624, |
| "logps/chosen": -91.15408325195312, |
| "logps/rejected": -130.5845489501953, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.3261685371398926, |
| "rewards/margins": 30.6134033203125, |
| "rewards/rejected": -27.287235260009766, |
| "step": 3490 |
| }, |
| { |
| "epoch": 1.6, |
| "learning_rate": 1.5575849822425165e-07, |
| "logits/chosen": -2.22548246383667, |
| "logits/rejected": -1.8935177326202393, |
| "logps/chosen": -83.39473724365234, |
| "logps/rejected": -121.51374816894531, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.055039882659912, |
| "rewards/margins": 27.6326904296875, |
| "rewards/rejected": -25.577648162841797, |
| "step": 3500 |
| }, |
| { |
| "epoch": 1.6, |
| "eval_logits/chosen": -2.186929941177368, |
| "eval_logits/rejected": -1.8434008359909058, |
| "eval_logps/chosen": -84.8189697265625, |
| "eval_logps/rejected": -120.2051010131836, |
| "eval_loss": 0.00656374916434288, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 2.0058064460754395, |
| "eval_rewards/margins": 28.69289779663086, |
| "eval_rewards/rejected": -26.687089920043945, |
| "eval_runtime": 188.198, |
| "eval_samples_per_second": 15.207, |
| "eval_steps_per_second": 0.951, |
| "step": 3500 |
| }, |
| { |
| "epoch": 1.6, |
| "learning_rate": 1.552511415525114e-07, |
| "logits/chosen": -2.2018935680389404, |
| "logits/rejected": -1.8334615230560303, |
| "logps/chosen": -85.780029296875, |
| "logps/rejected": -124.47122955322266, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9497463703155518, |
| "rewards/margins": 29.585214614868164, |
| "rewards/rejected": -27.635467529296875, |
| "step": 3510 |
| }, |
| { |
| "epoch": 1.61, |
| "learning_rate": 1.5474378488077118e-07, |
| "logits/chosen": -2.2138073444366455, |
| "logits/rejected": -1.819265365600586, |
| "logps/chosen": -91.44041442871094, |
| "logps/rejected": -128.78485107421875, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9119396209716797, |
| "rewards/margins": 30.014429092407227, |
| "rewards/rejected": -28.10248374938965, |
| "step": 3520 |
| }, |
| { |
| "epoch": 1.61, |
| "learning_rate": 1.5423642820903095e-07, |
| "logits/chosen": -2.1547751426696777, |
| "logits/rejected": -1.8329432010650635, |
| "logps/chosen": -84.93801879882812, |
| "logps/rejected": -124.8791275024414, |
| "loss": 0.0042, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.6016337871551514, |
| "rewards/margins": 29.17998695373535, |
| "rewards/rejected": -26.578350067138672, |
| "step": 3530 |
| }, |
| { |
| "epoch": 1.62, |
| "learning_rate": 1.537290715372907e-07, |
| "logits/chosen": -2.2604050636291504, |
| "logits/rejected": -1.8284223079681396, |
| "logps/chosen": -88.08625030517578, |
| "logps/rejected": -120.90476989746094, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.400468349456787, |
| "rewards/margins": 29.343048095703125, |
| "rewards/rejected": -26.942581176757812, |
| "step": 3540 |
| }, |
| { |
| "epoch": 1.62, |
| "learning_rate": 1.5322171486555048e-07, |
| "logits/chosen": -2.2389979362487793, |
| "logits/rejected": -1.8285210132598877, |
| "logps/chosen": -92.38504791259766, |
| "logps/rejected": -129.23291015625, |
| "loss": 0.0017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8334972858428955, |
| "rewards/margins": 31.94364356994629, |
| "rewards/rejected": -29.11014747619629, |
| "step": 3550 |
| }, |
| { |
| "epoch": 1.63, |
| "learning_rate": 1.5271435819381025e-07, |
| "logits/chosen": -2.2842297554016113, |
| "logits/rejected": -1.9232155084609985, |
| "logps/chosen": -88.47393035888672, |
| "logps/rejected": -126.19525146484375, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.37927508354187, |
| "rewards/margins": 30.97516441345215, |
| "rewards/rejected": -28.595890045166016, |
| "step": 3560 |
| }, |
| { |
| "epoch": 1.63, |
| "learning_rate": 1.5220700152207e-07, |
| "logits/chosen": -2.1726303100585938, |
| "logits/rejected": -1.7958831787109375, |
| "logps/chosen": -92.33584594726562, |
| "logps/rejected": -127.83787536621094, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8039255142211914, |
| "rewards/margins": 29.595382690429688, |
| "rewards/rejected": -27.791458129882812, |
| "step": 3570 |
| }, |
| { |
| "epoch": 1.63, |
| "learning_rate": 1.5169964485032978e-07, |
| "logits/chosen": -2.295196771621704, |
| "logits/rejected": -1.9108684062957764, |
| "logps/chosen": -86.93212890625, |
| "logps/rejected": -136.36459350585938, |
| "loss": 0.0016, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.010756015777588, |
| "rewards/margins": 32.493587493896484, |
| "rewards/rejected": -30.482830047607422, |
| "step": 3580 |
| }, |
| { |
| "epoch": 1.64, |
| "learning_rate": 1.5119228817858955e-07, |
| "logits/chosen": -2.2773728370666504, |
| "logits/rejected": -1.9154014587402344, |
| "logps/chosen": -85.64041900634766, |
| "logps/rejected": -127.74327087402344, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8653056621551514, |
| "rewards/margins": 31.216760635375977, |
| "rewards/rejected": -28.351455688476562, |
| "step": 3590 |
| }, |
| { |
| "epoch": 1.64, |
| "learning_rate": 1.506849315068493e-07, |
| "logits/chosen": -2.1455962657928467, |
| "logits/rejected": -1.849805474281311, |
| "logps/chosen": -87.38817596435547, |
| "logps/rejected": -123.53019714355469, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1171107292175293, |
| "rewards/margins": 29.911731719970703, |
| "rewards/rejected": -27.79462242126465, |
| "step": 3600 |
| }, |
| { |
| "epoch": 1.64, |
| "eval_logits/chosen": -2.1960811614990234, |
| "eval_logits/rejected": -1.8514564037322998, |
| "eval_logps/chosen": -85.992919921875, |
| "eval_logps/rejected": -124.32015228271484, |
| "eval_loss": 0.005746352486312389, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.4188352823257446, |
| "eval_rewards/margins": 30.1634521484375, |
| "eval_rewards/rejected": -28.744617462158203, |
| "eval_runtime": 179.5705, |
| "eval_samples_per_second": 15.938, |
| "eval_steps_per_second": 0.997, |
| "step": 3600 |
| }, |
| { |
| "epoch": 1.65, |
| "learning_rate": 1.5017757483510908e-07, |
| "logits/chosen": -2.1365771293640137, |
| "logits/rejected": -1.8313045501708984, |
| "logps/chosen": -85.9605712890625, |
| "logps/rejected": -131.0879364013672, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9845980405807495, |
| "rewards/margins": 30.611257553100586, |
| "rewards/rejected": -29.626659393310547, |
| "step": 3610 |
| }, |
| { |
| "epoch": 1.65, |
| "learning_rate": 1.4967021816336885e-07, |
| "logits/chosen": -2.197392225265503, |
| "logits/rejected": -1.8468068838119507, |
| "logps/chosen": -82.49998474121094, |
| "logps/rejected": -126.23731994628906, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3484909534454346, |
| "rewards/margins": 31.131275177001953, |
| "rewards/rejected": -29.78278160095215, |
| "step": 3620 |
| }, |
| { |
| "epoch": 1.66, |
| "learning_rate": 1.491628614916286e-07, |
| "logits/chosen": -2.269951343536377, |
| "logits/rejected": -1.8826881647109985, |
| "logps/chosen": -88.26679992675781, |
| "logps/rejected": -132.8121795654297, |
| "loss": 0.0056, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.3638824224472046, |
| "rewards/margins": 31.313467025756836, |
| "rewards/rejected": -29.949581146240234, |
| "step": 3630 |
| }, |
| { |
| "epoch": 1.66, |
| "learning_rate": 1.4865550481988838e-07, |
| "logits/chosen": -2.1884894371032715, |
| "logits/rejected": -1.8790266513824463, |
| "logps/chosen": -79.186279296875, |
| "logps/rejected": -124.1537094116211, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4217311143875122, |
| "rewards/margins": 30.239089965820312, |
| "rewards/rejected": -28.81736183166504, |
| "step": 3640 |
| }, |
| { |
| "epoch": 1.67, |
| "learning_rate": 1.4814814814814815e-07, |
| "logits/chosen": -2.2884726524353027, |
| "logits/rejected": -1.9434821605682373, |
| "logps/chosen": -82.31842041015625, |
| "logps/rejected": -121.79225158691406, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6052954196929932, |
| "rewards/margins": 30.475265502929688, |
| "rewards/rejected": -28.869970321655273, |
| "step": 3650 |
| }, |
| { |
| "epoch": 1.67, |
| "learning_rate": 1.476407914764079e-07, |
| "logits/chosen": -2.2113142013549805, |
| "logits/rejected": -1.858170747756958, |
| "logps/chosen": -88.19541931152344, |
| "logps/rejected": -128.73983764648438, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.839066505432129, |
| "rewards/margins": 30.854511260986328, |
| "rewards/rejected": -29.015445709228516, |
| "step": 3660 |
| }, |
| { |
| "epoch": 1.68, |
| "learning_rate": 1.4713343480466768e-07, |
| "logits/chosen": -2.1949267387390137, |
| "logits/rejected": -1.8198333978652954, |
| "logps/chosen": -86.29945373535156, |
| "logps/rejected": -126.752197265625, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4010677337646484, |
| "rewards/margins": 32.765567779541016, |
| "rewards/rejected": -29.3644962310791, |
| "step": 3670 |
| }, |
| { |
| "epoch": 1.68, |
| "learning_rate": 1.4662607813292745e-07, |
| "logits/chosen": -2.2420246601104736, |
| "logits/rejected": -1.8449939489364624, |
| "logps/chosen": -88.93192291259766, |
| "logps/rejected": -125.01644134521484, |
| "loss": 0.0099, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.367809295654297, |
| "rewards/margins": 31.084781646728516, |
| "rewards/rejected": -28.716970443725586, |
| "step": 3680 |
| }, |
| { |
| "epoch": 1.68, |
| "learning_rate": 1.461187214611872e-07, |
| "logits/chosen": -2.1443724632263184, |
| "logits/rejected": -1.7969995737075806, |
| "logps/chosen": -88.4251937866211, |
| "logps/rejected": -124.62715911865234, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7502880096435547, |
| "rewards/margins": 30.875295639038086, |
| "rewards/rejected": -28.125009536743164, |
| "step": 3690 |
| }, |
| { |
| "epoch": 1.69, |
| "learning_rate": 1.4561136478944698e-07, |
| "logits/chosen": -2.1906471252441406, |
| "logits/rejected": -1.8091493844985962, |
| "logps/chosen": -85.97318267822266, |
| "logps/rejected": -130.11227416992188, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.8357086181640625, |
| "rewards/margins": 32.3084602355957, |
| "rewards/rejected": -28.47275161743164, |
| "step": 3700 |
| }, |
| { |
| "epoch": 1.69, |
| "eval_logits/chosen": -2.1971521377563477, |
| "eval_logits/rejected": -1.848021149635315, |
| "eval_logps/chosen": -84.61711120605469, |
| "eval_logps/rejected": -123.59754943847656, |
| "eval_loss": 0.005367867648601532, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.106738567352295, |
| "eval_rewards/margins": 30.49005699157715, |
| "eval_rewards/rejected": -28.38331413269043, |
| "eval_runtime": 176.7324, |
| "eval_samples_per_second": 16.194, |
| "eval_steps_per_second": 1.013, |
| "step": 3700 |
| }, |
| { |
| "epoch": 1.69, |
| "learning_rate": 1.4510400811770675e-07, |
| "logits/chosen": -2.226139545440674, |
| "logits/rejected": -1.836024522781372, |
| "logps/chosen": -91.7187271118164, |
| "logps/rejected": -129.92689514160156, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7738643884658813, |
| "rewards/margins": 30.75638198852539, |
| "rewards/rejected": -28.982519149780273, |
| "step": 3710 |
| }, |
| { |
| "epoch": 1.7, |
| "learning_rate": 1.445966514459665e-07, |
| "logits/chosen": -2.2146477699279785, |
| "logits/rejected": -1.8724933862686157, |
| "logps/chosen": -82.95762634277344, |
| "logps/rejected": -124.72331237792969, |
| "loss": 0.005, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.609710216522217, |
| "rewards/margins": 30.03472328186035, |
| "rewards/rejected": -27.425012588500977, |
| "step": 3720 |
| }, |
| { |
| "epoch": 1.7, |
| "learning_rate": 1.4408929477422628e-07, |
| "logits/chosen": -2.1765360832214355, |
| "logits/rejected": -1.8613145351409912, |
| "logps/chosen": -88.75824737548828, |
| "logps/rejected": -126.1810531616211, |
| "loss": 0.0056, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3347582817077637, |
| "rewards/margins": 30.444360733032227, |
| "rewards/rejected": -28.109600067138672, |
| "step": 3730 |
| }, |
| { |
| "epoch": 1.71, |
| "learning_rate": 1.4358193810248604e-07, |
| "logits/chosen": -2.203244686126709, |
| "logits/rejected": -1.8518617153167725, |
| "logps/chosen": -83.48390197753906, |
| "logps/rejected": -124.0763168334961, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4211070537567139, |
| "rewards/margins": 29.72439956665039, |
| "rewards/rejected": -28.303295135498047, |
| "step": 3740 |
| }, |
| { |
| "epoch": 1.71, |
| "learning_rate": 1.430745814307458e-07, |
| "logits/chosen": -2.231482982635498, |
| "logits/rejected": -1.8217432498931885, |
| "logps/chosen": -89.63983917236328, |
| "logps/rejected": -125.65821838378906, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.655423641204834, |
| "rewards/margins": 29.783100128173828, |
| "rewards/rejected": -27.127676010131836, |
| "step": 3750 |
| }, |
| { |
| "epoch": 1.72, |
| "learning_rate": 1.4256722475900558e-07, |
| "logits/chosen": -2.152547597885132, |
| "logits/rejected": -1.8171924352645874, |
| "logps/chosen": -81.50759887695312, |
| "logps/rejected": -123.8464126586914, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3097615242004395, |
| "rewards/margins": 29.635555267333984, |
| "rewards/rejected": -28.325796127319336, |
| "step": 3760 |
| }, |
| { |
| "epoch": 1.72, |
| "learning_rate": 1.4205986808726534e-07, |
| "logits/chosen": -2.249340534210205, |
| "logits/rejected": -1.9595706462860107, |
| "logps/chosen": -85.8914794921875, |
| "logps/rejected": -138.0444793701172, |
| "loss": 0.0007, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.460918664932251, |
| "rewards/margins": 32.2510871887207, |
| "rewards/rejected": -29.790172576904297, |
| "step": 3770 |
| }, |
| { |
| "epoch": 1.73, |
| "learning_rate": 1.415525114155251e-07, |
| "logits/chosen": -2.1925182342529297, |
| "logits/rejected": -1.7810020446777344, |
| "logps/chosen": -93.9482421875, |
| "logps/rejected": -130.27859497070312, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.997776746749878, |
| "rewards/margins": 31.849191665649414, |
| "rewards/rejected": -29.851415634155273, |
| "step": 3780 |
| }, |
| { |
| "epoch": 1.73, |
| "learning_rate": 1.4104515474378488e-07, |
| "logits/chosen": -2.134547472000122, |
| "logits/rejected": -1.8135312795639038, |
| "logps/chosen": -84.8584976196289, |
| "logps/rejected": -123.8740005493164, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.4246654510498047, |
| "rewards/margins": 29.43796730041504, |
| "rewards/rejected": -28.013301849365234, |
| "step": 3790 |
| }, |
| { |
| "epoch": 1.73, |
| "learning_rate": 1.4053779807204464e-07, |
| "logits/chosen": -2.267373561859131, |
| "logits/rejected": -1.8483003377914429, |
| "logps/chosen": -87.11589050292969, |
| "logps/rejected": -115.36067199707031, |
| "loss": 0.006, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.7466697692871094, |
| "rewards/margins": 28.394153594970703, |
| "rewards/rejected": -24.647480010986328, |
| "step": 3800 |
| }, |
| { |
| "epoch": 1.73, |
| "eval_logits/chosen": -2.191563367843628, |
| "eval_logits/rejected": -1.8433810472488403, |
| "eval_logps/chosen": -83.24746704101562, |
| "eval_logps/rejected": -116.2911148071289, |
| "eval_loss": 0.005436885170638561, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.791560173034668, |
| "eval_rewards/margins": 27.521656036376953, |
| "eval_rewards/rejected": -24.73009490966797, |
| "eval_runtime": 221.0913, |
| "eval_samples_per_second": 12.945, |
| "eval_steps_per_second": 0.81, |
| "step": 3800 |
| }, |
| { |
| "epoch": 1.74, |
| "learning_rate": 1.400304414003044e-07, |
| "logits/chosen": -2.2577452659606934, |
| "logits/rejected": -1.8542404174804688, |
| "logps/chosen": -81.37186431884766, |
| "logps/rejected": -114.3775405883789, |
| "loss": 0.0011, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.739640951156616, |
| "rewards/margins": 26.94769859313965, |
| "rewards/rejected": -24.208059310913086, |
| "step": 3810 |
| }, |
| { |
| "epoch": 1.74, |
| "learning_rate": 1.3952308472856418e-07, |
| "logits/chosen": -2.1801652908325195, |
| "logits/rejected": -1.8121554851531982, |
| "logps/chosen": -87.84750366210938, |
| "logps/rejected": -119.56182861328125, |
| "loss": 0.0007, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7155660390853882, |
| "rewards/margins": 28.609859466552734, |
| "rewards/rejected": -26.894290924072266, |
| "step": 3820 |
| }, |
| { |
| "epoch": 1.75, |
| "learning_rate": 1.3901572805682394e-07, |
| "logits/chosen": -2.166355609893799, |
| "logits/rejected": -1.7864339351654053, |
| "logps/chosen": -87.84122467041016, |
| "logps/rejected": -126.62345886230469, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1120693683624268, |
| "rewards/margins": 30.040283203125, |
| "rewards/rejected": -26.928213119506836, |
| "step": 3830 |
| }, |
| { |
| "epoch": 1.75, |
| "learning_rate": 1.385083713850837e-07, |
| "logits/chosen": -2.1811468601226807, |
| "logits/rejected": -1.7890942096710205, |
| "logps/chosen": -86.22557830810547, |
| "logps/rejected": -121.93861389160156, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1839969158172607, |
| "rewards/margins": 29.871551513671875, |
| "rewards/rejected": -26.687557220458984, |
| "step": 3840 |
| }, |
| { |
| "epoch": 1.76, |
| "learning_rate": 1.3800101471334348e-07, |
| "logits/chosen": -2.186249256134033, |
| "logits/rejected": -1.8273032903671265, |
| "logps/chosen": -86.01737213134766, |
| "logps/rejected": -119.9991226196289, |
| "loss": 0.0055, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.2794878482818604, |
| "rewards/margins": 28.3900203704834, |
| "rewards/rejected": -27.11053466796875, |
| "step": 3850 |
| }, |
| { |
| "epoch": 1.76, |
| "learning_rate": 1.3749365804160324e-07, |
| "logits/chosen": -2.122135639190674, |
| "logits/rejected": -1.7545080184936523, |
| "logps/chosen": -86.4559555053711, |
| "logps/rejected": -118.87664794921875, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.654794454574585, |
| "rewards/margins": 27.156728744506836, |
| "rewards/rejected": -25.501934051513672, |
| "step": 3860 |
| }, |
| { |
| "epoch": 1.77, |
| "learning_rate": 1.36986301369863e-07, |
| "logits/chosen": -2.1122047901153564, |
| "logits/rejected": -1.7238849401474, |
| "logps/chosen": -84.66944122314453, |
| "logps/rejected": -121.77825927734375, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.5499777793884277, |
| "rewards/margins": 31.206939697265625, |
| "rewards/rejected": -27.656963348388672, |
| "step": 3870 |
| }, |
| { |
| "epoch": 1.77, |
| "learning_rate": 1.3647894469812278e-07, |
| "logits/chosen": -2.1466057300567627, |
| "logits/rejected": -1.7561490535736084, |
| "logps/chosen": -84.68426513671875, |
| "logps/rejected": -123.66081237792969, |
| "loss": 0.0014, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.3084232807159424, |
| "rewards/margins": 31.223583221435547, |
| "rewards/rejected": -27.915157318115234, |
| "step": 3880 |
| }, |
| { |
| "epoch": 1.78, |
| "learning_rate": 1.3597158802638254e-07, |
| "logits/chosen": -2.1600029468536377, |
| "logits/rejected": -1.7710784673690796, |
| "logps/chosen": -87.58964538574219, |
| "logps/rejected": -127.94322204589844, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7342724800109863, |
| "rewards/margins": 30.390094757080078, |
| "rewards/rejected": -27.65582275390625, |
| "step": 3890 |
| }, |
| { |
| "epoch": 1.78, |
| "learning_rate": 1.354642313546423e-07, |
| "logits/chosen": -2.1588668823242188, |
| "logits/rejected": -1.8171707391738892, |
| "logps/chosen": -82.0718002319336, |
| "logps/rejected": -117.87044525146484, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.580735445022583, |
| "rewards/margins": 28.978771209716797, |
| "rewards/rejected": -26.398035049438477, |
| "step": 3900 |
| }, |
| { |
| "epoch": 1.78, |
| "eval_logits/chosen": -2.1919665336608887, |
| "eval_logits/rejected": -1.8445065021514893, |
| "eval_logps/chosen": -84.54802703857422, |
| "eval_logps/rejected": -120.56312561035156, |
| "eval_loss": 0.005169562995433807, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 2.1412765979766846, |
| "eval_rewards/margins": 29.007375717163086, |
| "eval_rewards/rejected": -26.866098403930664, |
| "eval_runtime": 203.0484, |
| "eval_samples_per_second": 14.095, |
| "eval_steps_per_second": 0.882, |
| "step": 3900 |
| }, |
| { |
| "epoch": 1.78, |
| "learning_rate": 1.3495687468290208e-07, |
| "logits/chosen": -2.2244656085968018, |
| "logits/rejected": -1.810752272605896, |
| "logps/chosen": -89.08476257324219, |
| "logps/rejected": -123.11407470703125, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.44339919090271, |
| "rewards/margins": 30.2976016998291, |
| "rewards/rejected": -27.854202270507812, |
| "step": 3910 |
| }, |
| { |
| "epoch": 1.79, |
| "learning_rate": 1.3444951801116184e-07, |
| "logits/chosen": -2.219104290008545, |
| "logits/rejected": -1.8220125436782837, |
| "logps/chosen": -89.28169250488281, |
| "logps/rejected": -126.15754699707031, |
| "loss": 0.0039, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9085346460342407, |
| "rewards/margins": 30.235088348388672, |
| "rewards/rejected": -28.326553344726562, |
| "step": 3920 |
| }, |
| { |
| "epoch": 1.79, |
| "learning_rate": 1.339421613394216e-07, |
| "logits/chosen": -2.276082754135132, |
| "logits/rejected": -1.8841686248779297, |
| "logps/chosen": -89.07032775878906, |
| "logps/rejected": -126.38359069824219, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.427870988845825, |
| "rewards/margins": 30.165613174438477, |
| "rewards/rejected": -27.737743377685547, |
| "step": 3930 |
| }, |
| { |
| "epoch": 1.8, |
| "learning_rate": 1.3343480466768138e-07, |
| "logits/chosen": -2.227997064590454, |
| "logits/rejected": -1.8480758666992188, |
| "logps/chosen": -84.52095794677734, |
| "logps/rejected": -119.73951721191406, |
| "loss": 0.0066, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.0445349216461182, |
| "rewards/margins": 28.596317291259766, |
| "rewards/rejected": -27.551782608032227, |
| "step": 3940 |
| }, |
| { |
| "epoch": 1.8, |
| "learning_rate": 1.3292744799594114e-07, |
| "logits/chosen": -2.182831287384033, |
| "logits/rejected": -1.7697114944458008, |
| "logps/chosen": -88.89148712158203, |
| "logps/rejected": -125.78935241699219, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.283107042312622, |
| "rewards/margins": 31.58938217163086, |
| "rewards/rejected": -28.306278228759766, |
| "step": 3950 |
| }, |
| { |
| "epoch": 1.81, |
| "learning_rate": 1.324200913242009e-07, |
| "logits/chosen": -2.156838893890381, |
| "logits/rejected": -1.7892711162567139, |
| "logps/chosen": -82.74298095703125, |
| "logps/rejected": -127.2794189453125, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.570988416671753, |
| "rewards/margins": 32.732200622558594, |
| "rewards/rejected": -29.161212921142578, |
| "step": 3960 |
| }, |
| { |
| "epoch": 1.81, |
| "learning_rate": 1.3191273465246068e-07, |
| "logits/chosen": -2.2453393936157227, |
| "logits/rejected": -1.8167974948883057, |
| "logps/chosen": -89.5159912109375, |
| "logps/rejected": -128.8140869140625, |
| "loss": 0.0068, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.0265374183654785, |
| "rewards/margins": 31.421884536743164, |
| "rewards/rejected": -28.395349502563477, |
| "step": 3970 |
| }, |
| { |
| "epoch": 1.82, |
| "learning_rate": 1.3140537798072044e-07, |
| "logits/chosen": -2.1521337032318115, |
| "logits/rejected": -1.7532793283462524, |
| "logps/chosen": -85.2146987915039, |
| "logps/rejected": -111.48863220214844, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.89916729927063, |
| "rewards/margins": 27.08025550842285, |
| "rewards/rejected": -23.181087493896484, |
| "step": 3980 |
| }, |
| { |
| "epoch": 1.82, |
| "learning_rate": 1.308980213089802e-07, |
| "logits/chosen": -2.2195143699645996, |
| "logits/rejected": -1.8655322790145874, |
| "logps/chosen": -83.930419921875, |
| "logps/rejected": -112.97065734863281, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.9724647998809814, |
| "rewards/margins": 25.749919891357422, |
| "rewards/rejected": -21.777454376220703, |
| "step": 3990 |
| }, |
| { |
| "epoch": 1.83, |
| "learning_rate": 1.3039066463723998e-07, |
| "logits/chosen": -2.243203639984131, |
| "logits/rejected": -1.7824580669403076, |
| "logps/chosen": -85.41126251220703, |
| "logps/rejected": -116.12227630615234, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.609849691390991, |
| "rewards/margins": 27.11124038696289, |
| "rewards/rejected": -24.50139045715332, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.83, |
| "eval_logits/chosen": -2.207897663116455, |
| "eval_logits/rejected": -1.8570655584335327, |
| "eval_logps/chosen": -83.74642181396484, |
| "eval_logps/rejected": -115.08486938476562, |
| "eval_loss": 0.0051609063521027565, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 2.5420799255371094, |
| "eval_rewards/margins": 26.66905403137207, |
| "eval_rewards/rejected": -24.126972198486328, |
| "eval_runtime": 213.5807, |
| "eval_samples_per_second": 13.4, |
| "eval_steps_per_second": 0.838, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.83, |
| "learning_rate": 1.2988330796549974e-07, |
| "logits/chosen": -2.2335009574890137, |
| "logits/rejected": -1.8103595972061157, |
| "logps/chosen": -87.22106170654297, |
| "logps/rejected": -117.1398696899414, |
| "loss": 0.0065, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 2.971987247467041, |
| "rewards/margins": 27.090845108032227, |
| "rewards/rejected": -24.118860244750977, |
| "step": 4010 |
| }, |
| { |
| "epoch": 1.83, |
| "learning_rate": 1.293759512937595e-07, |
| "logits/chosen": -2.2131710052490234, |
| "logits/rejected": -1.8810796737670898, |
| "logps/chosen": -89.93437957763672, |
| "logps/rejected": -119.86589050292969, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9654256701469421, |
| "rewards/margins": 24.501127243041992, |
| "rewards/rejected": -23.535701751708984, |
| "step": 4020 |
| }, |
| { |
| "epoch": 1.84, |
| "learning_rate": 1.2886859462201928e-07, |
| "logits/chosen": -2.2927210330963135, |
| "logits/rejected": -1.9104375839233398, |
| "logps/chosen": -88.72476196289062, |
| "logps/rejected": -117.77976989746094, |
| "loss": 0.0041, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.68806791305542, |
| "rewards/margins": 27.564483642578125, |
| "rewards/rejected": -24.876415252685547, |
| "step": 4030 |
| }, |
| { |
| "epoch": 1.84, |
| "learning_rate": 1.2836123795027904e-07, |
| "logits/chosen": -2.168994903564453, |
| "logits/rejected": -1.7794716358184814, |
| "logps/chosen": -81.93544006347656, |
| "logps/rejected": -116.9870376586914, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4068737030029297, |
| "rewards/margins": 27.525577545166016, |
| "rewards/rejected": -25.118701934814453, |
| "step": 4040 |
| }, |
| { |
| "epoch": 1.85, |
| "learning_rate": 1.278538812785388e-07, |
| "logits/chosen": -2.18416690826416, |
| "logits/rejected": -1.7201156616210938, |
| "logps/chosen": -91.39389038085938, |
| "logps/rejected": -118.92759704589844, |
| "loss": 0.0063, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.148336887359619, |
| "rewards/margins": 27.875076293945312, |
| "rewards/rejected": -24.72673988342285, |
| "step": 4050 |
| }, |
| { |
| "epoch": 1.85, |
| "learning_rate": 1.2734652460679858e-07, |
| "logits/chosen": -2.2627367973327637, |
| "logits/rejected": -1.9123704433441162, |
| "logps/chosen": -88.3521728515625, |
| "logps/rejected": -117.85545349121094, |
| "loss": 0.0027, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9983296394348145, |
| "rewards/margins": 25.974822998046875, |
| "rewards/rejected": -22.976491928100586, |
| "step": 4060 |
| }, |
| { |
| "epoch": 1.86, |
| "learning_rate": 1.2683916793505834e-07, |
| "logits/chosen": -2.1818766593933105, |
| "logits/rejected": -1.8303353786468506, |
| "logps/chosen": -82.59492492675781, |
| "logps/rejected": -120.16998291015625, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4224205017089844, |
| "rewards/margins": 27.243423461914062, |
| "rewards/rejected": -23.821001052856445, |
| "step": 4070 |
| }, |
| { |
| "epoch": 1.86, |
| "learning_rate": 1.263318112633181e-07, |
| "logits/chosen": -2.2209744453430176, |
| "logits/rejected": -1.8506110906600952, |
| "logps/chosen": -87.74772644042969, |
| "logps/rejected": -112.5324478149414, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.3162944316864014, |
| "rewards/margins": 25.54704475402832, |
| "rewards/rejected": -22.230749130249023, |
| "step": 4080 |
| }, |
| { |
| "epoch": 1.87, |
| "learning_rate": 1.2582445459157788e-07, |
| "logits/chosen": -2.203535318374634, |
| "logits/rejected": -1.8149940967559814, |
| "logps/chosen": -81.7652587890625, |
| "logps/rejected": -113.93977355957031, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2748992443084717, |
| "rewards/margins": 26.76283836364746, |
| "rewards/rejected": -23.48794174194336, |
| "step": 4090 |
| }, |
| { |
| "epoch": 1.87, |
| "learning_rate": 1.2531709791983764e-07, |
| "logits/chosen": -2.271902561187744, |
| "logits/rejected": -1.8522933721542358, |
| "logps/chosen": -89.37789916992188, |
| "logps/rejected": -117.07502746582031, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4491279125213623, |
| "rewards/margins": 26.383209228515625, |
| "rewards/rejected": -22.934078216552734, |
| "step": 4100 |
| }, |
| { |
| "epoch": 1.87, |
| "eval_logits/chosen": -2.217360019683838, |
| "eval_logits/rejected": -1.8673908710479736, |
| "eval_logps/chosen": -83.58930969238281, |
| "eval_logps/rejected": -114.42141723632812, |
| "eval_loss": 0.005190215539187193, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.6206393241882324, |
| "eval_rewards/margins": 26.415889739990234, |
| "eval_rewards/rejected": -23.795251846313477, |
| "eval_runtime": 206.2078, |
| "eval_samples_per_second": 13.879, |
| "eval_steps_per_second": 0.868, |
| "step": 4100 |
| }, |
| { |
| "epoch": 1.88, |
| "learning_rate": 1.248097412480974e-07, |
| "logits/chosen": -2.3479442596435547, |
| "logits/rejected": -1.9204511642456055, |
| "logps/chosen": -88.93357849121094, |
| "logps/rejected": -119.96925354003906, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.7320475578308105, |
| "rewards/margins": 28.458393096923828, |
| "rewards/rejected": -24.72634506225586, |
| "step": 4110 |
| }, |
| { |
| "epoch": 1.88, |
| "learning_rate": 1.2430238457635718e-07, |
| "logits/chosen": -2.1347765922546387, |
| "logits/rejected": -1.743198037147522, |
| "logps/chosen": -87.92202758789062, |
| "logps/rejected": -118.41119384765625, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6807589530944824, |
| "rewards/margins": 27.0130558013916, |
| "rewards/rejected": -24.33229637145996, |
| "step": 4120 |
| }, |
| { |
| "epoch": 1.89, |
| "learning_rate": 1.2379502790461694e-07, |
| "logits/chosen": -2.176631450653076, |
| "logits/rejected": -1.8327052593231201, |
| "logps/chosen": -84.86781311035156, |
| "logps/rejected": -117.3744888305664, |
| "loss": 0.0054, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2650296688079834, |
| "rewards/margins": 25.868539810180664, |
| "rewards/rejected": -24.603511810302734, |
| "step": 4130 |
| }, |
| { |
| "epoch": 1.89, |
| "learning_rate": 1.232876712328767e-07, |
| "logits/chosen": -2.158825635910034, |
| "logits/rejected": -1.7359037399291992, |
| "logps/chosen": -88.40345001220703, |
| "logps/rejected": -119.45426177978516, |
| "loss": 0.0016, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4086227416992188, |
| "rewards/margins": 27.16936683654785, |
| "rewards/rejected": -24.760744094848633, |
| "step": 4140 |
| }, |
| { |
| "epoch": 1.89, |
| "learning_rate": 1.2278031456113648e-07, |
| "logits/chosen": -2.2729620933532715, |
| "logits/rejected": -1.8309637308120728, |
| "logps/chosen": -91.2960205078125, |
| "logps/rejected": -117.98994445800781, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.7180824279785156, |
| "rewards/margins": 27.0567569732666, |
| "rewards/rejected": -23.33867645263672, |
| "step": 4150 |
| }, |
| { |
| "epoch": 1.9, |
| "learning_rate": 1.2227295788939624e-07, |
| "logits/chosen": -2.2085700035095215, |
| "logits/rejected": -1.8833599090576172, |
| "logps/chosen": -86.73484802246094, |
| "logps/rejected": -124.00797271728516, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6060290336608887, |
| "rewards/margins": 27.345844268798828, |
| "rewards/rejected": -24.739816665649414, |
| "step": 4160 |
| }, |
| { |
| "epoch": 1.9, |
| "learning_rate": 1.21765601217656e-07, |
| "logits/chosen": -2.231919050216675, |
| "logits/rejected": -1.8927338123321533, |
| "logps/chosen": -82.05496978759766, |
| "logps/rejected": -125.29072570800781, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5094146728515625, |
| "rewards/margins": 27.967309951782227, |
| "rewards/rejected": -25.457895278930664, |
| "step": 4170 |
| }, |
| { |
| "epoch": 1.91, |
| "learning_rate": 1.2125824454591578e-07, |
| "logits/chosen": -2.252375364303589, |
| "logits/rejected": -1.7970874309539795, |
| "logps/chosen": -92.34709167480469, |
| "logps/rejected": -115.58805084228516, |
| "loss": 0.0047, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9096739292144775, |
| "rewards/margins": 27.263355255126953, |
| "rewards/rejected": -24.353681564331055, |
| "step": 4180 |
| }, |
| { |
| "epoch": 1.91, |
| "learning_rate": 1.2075088787417554e-07, |
| "logits/chosen": -2.1913251876831055, |
| "logits/rejected": -1.8256721496582031, |
| "logps/chosen": -83.70695495605469, |
| "logps/rejected": -117.19889068603516, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.558323621749878, |
| "rewards/margins": 27.682703018188477, |
| "rewards/rejected": -24.124378204345703, |
| "step": 4190 |
| }, |
| { |
| "epoch": 1.92, |
| "learning_rate": 1.202435312024353e-07, |
| "logits/chosen": -2.2380259037017822, |
| "logits/rejected": -1.9088646173477173, |
| "logps/chosen": -82.8735580444336, |
| "logps/rejected": -115.74534606933594, |
| "loss": 0.0026, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.5881214141845703, |
| "rewards/margins": 26.207622528076172, |
| "rewards/rejected": -23.6195011138916, |
| "step": 4200 |
| }, |
| { |
| "epoch": 1.92, |
| "eval_logits/chosen": -2.2144737243652344, |
| "eval_logits/rejected": -1.8624593019485474, |
| "eval_logps/chosen": -83.490234375, |
| "eval_logps/rejected": -111.31690979003906, |
| "eval_loss": 0.00542183592915535, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.670175075531006, |
| "eval_rewards/margins": 24.913169860839844, |
| "eval_rewards/rejected": -22.242996215820312, |
| "eval_runtime": 204.9075, |
| "eval_samples_per_second": 13.967, |
| "eval_steps_per_second": 0.874, |
| "step": 4200 |
| }, |
| { |
| "epoch": 1.92, |
| "learning_rate": 1.1973617453069508e-07, |
| "logits/chosen": -2.214597463607788, |
| "logits/rejected": -1.8877366781234741, |
| "logps/chosen": -81.5813980102539, |
| "logps/rejected": -110.80888366699219, |
| "loss": 0.0036, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.2836194038391113, |
| "rewards/margins": 24.602767944335938, |
| "rewards/rejected": -22.31914710998535, |
| "step": 4210 |
| }, |
| { |
| "epoch": 1.93, |
| "learning_rate": 1.1922881785895484e-07, |
| "logits/chosen": -2.2340848445892334, |
| "logits/rejected": -1.8911195993423462, |
| "logps/chosen": -88.96124267578125, |
| "logps/rejected": -121.70379638671875, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.17574405670166, |
| "rewards/margins": 26.648061752319336, |
| "rewards/rejected": -23.47231674194336, |
| "step": 4220 |
| }, |
| { |
| "epoch": 1.93, |
| "learning_rate": 1.187214611872146e-07, |
| "logits/chosen": -2.306396007537842, |
| "logits/rejected": -1.935389757156372, |
| "logps/chosen": -90.6281509399414, |
| "logps/rejected": -122.7242660522461, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.553041934967041, |
| "rewards/margins": 26.54819107055664, |
| "rewards/rejected": -23.995147705078125, |
| "step": 4230 |
| }, |
| { |
| "epoch": 1.94, |
| "learning_rate": 1.1821410451547436e-07, |
| "logits/chosen": -2.159374713897705, |
| "logits/rejected": -1.8598381280899048, |
| "logps/chosen": -81.08094024658203, |
| "logps/rejected": -112.6502914428711, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.01515531539917, |
| "rewards/margins": 25.243701934814453, |
| "rewards/rejected": -21.22854232788086, |
| "step": 4240 |
| }, |
| { |
| "epoch": 1.94, |
| "learning_rate": 1.1770674784373413e-07, |
| "logits/chosen": -2.15578031539917, |
| "logits/rejected": -1.851265549659729, |
| "logps/chosen": -78.29918670654297, |
| "logps/rejected": -111.49513244628906, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1447842121124268, |
| "rewards/margins": 23.960596084594727, |
| "rewards/rejected": -21.815811157226562, |
| "step": 4250 |
| }, |
| { |
| "epoch": 1.94, |
| "learning_rate": 1.171993911719939e-07, |
| "logits/chosen": -2.2845335006713867, |
| "logits/rejected": -1.9803342819213867, |
| "logps/chosen": -84.60355377197266, |
| "logps/rejected": -118.61265563964844, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0940170288085938, |
| "rewards/margins": 26.21561050415039, |
| "rewards/rejected": -24.121593475341797, |
| "step": 4260 |
| }, |
| { |
| "epoch": 1.95, |
| "learning_rate": 1.1669203450025366e-07, |
| "logits/chosen": -2.1469063758850098, |
| "logits/rejected": -1.885765790939331, |
| "logps/chosen": -78.9441909790039, |
| "logps/rejected": -114.021240234375, |
| "loss": 0.0073, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.2761025428771973, |
| "rewards/margins": 24.522062301635742, |
| "rewards/rejected": -22.24595832824707, |
| "step": 4270 |
| }, |
| { |
| "epoch": 1.95, |
| "learning_rate": 1.1618467782851343e-07, |
| "logits/chosen": -2.2316300868988037, |
| "logits/rejected": -1.8505769968032837, |
| "logps/chosen": -82.11341857910156, |
| "logps/rejected": -119.51060485839844, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4899394512176514, |
| "rewards/margins": 26.23581314086914, |
| "rewards/rejected": -23.74587059020996, |
| "step": 4280 |
| }, |
| { |
| "epoch": 1.96, |
| "learning_rate": 1.156773211567732e-07, |
| "logits/chosen": -2.1731178760528564, |
| "logits/rejected": -1.7443568706512451, |
| "logps/chosen": -84.8414306640625, |
| "logps/rejected": -119.10728454589844, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4432387351989746, |
| "rewards/margins": 28.680456161499023, |
| "rewards/rejected": -25.237218856811523, |
| "step": 4290 |
| }, |
| { |
| "epoch": 1.96, |
| "learning_rate": 1.1516996448503296e-07, |
| "logits/chosen": -2.245872974395752, |
| "logits/rejected": -1.8761298656463623, |
| "logps/chosen": -83.03449249267578, |
| "logps/rejected": -122.46466064453125, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.476714611053467, |
| "rewards/margins": 27.374313354492188, |
| "rewards/rejected": -24.897600173950195, |
| "step": 4300 |
| }, |
| { |
| "epoch": 1.96, |
| "eval_logits/chosen": -2.2145586013793945, |
| "eval_logits/rejected": -1.862236499786377, |
| "eval_logps/chosen": -83.82196807861328, |
| "eval_logps/rejected": -114.24178314208984, |
| "eval_loss": 0.005356738809496164, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 2.504307270050049, |
| "eval_rewards/margins": 26.20973777770996, |
| "eval_rewards/rejected": -23.705427169799805, |
| "eval_runtime": 231.7023, |
| "eval_samples_per_second": 12.352, |
| "eval_steps_per_second": 0.773, |
| "step": 4300 |
| }, |
| { |
| "epoch": 1.97, |
| "learning_rate": 1.1466260781329273e-07, |
| "logits/chosen": -2.2044477462768555, |
| "logits/rejected": -1.769928216934204, |
| "logps/chosen": -87.90011596679688, |
| "logps/rejected": -117.90144348144531, |
| "loss": 0.0021, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1481876373291016, |
| "rewards/margins": 27.33310317993164, |
| "rewards/rejected": -24.184917449951172, |
| "step": 4310 |
| }, |
| { |
| "epoch": 1.97, |
| "learning_rate": 1.141552511415525e-07, |
| "logits/chosen": -2.2101001739501953, |
| "logits/rejected": -1.8445708751678467, |
| "logps/chosen": -90.51927185058594, |
| "logps/rejected": -117.70124816894531, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.421943426132202, |
| "rewards/margins": 27.004901885986328, |
| "rewards/rejected": -24.582958221435547, |
| "step": 4320 |
| }, |
| { |
| "epoch": 1.98, |
| "learning_rate": 1.1364789446981226e-07, |
| "logits/chosen": -2.20845365524292, |
| "logits/rejected": -1.8483657836914062, |
| "logps/chosen": -90.68830871582031, |
| "logps/rejected": -119.9162368774414, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.908290386199951, |
| "rewards/margins": 27.960651397705078, |
| "rewards/rejected": -25.052364349365234, |
| "step": 4330 |
| }, |
| { |
| "epoch": 1.98, |
| "learning_rate": 1.1314053779807203e-07, |
| "logits/chosen": -2.2378525733947754, |
| "logits/rejected": -1.9095252752304077, |
| "logps/chosen": -84.37041473388672, |
| "logps/rejected": -117.41896057128906, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9735949039459229, |
| "rewards/margins": 26.64251136779785, |
| "rewards/rejected": -24.668912887573242, |
| "step": 4340 |
| }, |
| { |
| "epoch": 1.99, |
| "learning_rate": 1.126331811263318e-07, |
| "logits/chosen": -2.2925407886505127, |
| "logits/rejected": -1.940437912940979, |
| "logps/chosen": -83.49569702148438, |
| "logps/rejected": -115.74214172363281, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6832937002182007, |
| "rewards/margins": 25.954483032226562, |
| "rewards/rejected": -24.271190643310547, |
| "step": 4350 |
| }, |
| { |
| "epoch": 1.99, |
| "learning_rate": 1.1212582445459156e-07, |
| "logits/chosen": -2.1880042552948, |
| "logits/rejected": -1.7285563945770264, |
| "logps/chosen": -90.88077545166016, |
| "logps/rejected": -118.75814056396484, |
| "loss": 0.0021, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5746841430664062, |
| "rewards/margins": 28.058208465576172, |
| "rewards/rejected": -25.483524322509766, |
| "step": 4360 |
| }, |
| { |
| "epoch": 1.99, |
| "learning_rate": 1.1161846778285133e-07, |
| "logits/chosen": -2.1667580604553223, |
| "logits/rejected": -1.878365159034729, |
| "logps/chosen": -78.63961029052734, |
| "logps/rejected": -118.59150695800781, |
| "loss": 0.0068, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0068283081054688, |
| "rewards/margins": 25.848861694335938, |
| "rewards/rejected": -23.8420352935791, |
| "step": 4370 |
| }, |
| { |
| "epoch": 2.0, |
| "learning_rate": 1.111111111111111e-07, |
| "logits/chosen": -2.160161018371582, |
| "logits/rejected": -1.7247244119644165, |
| "logps/chosen": -90.16160583496094, |
| "logps/rejected": -113.54561614990234, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1644601821899414, |
| "rewards/margins": 26.249164581298828, |
| "rewards/rejected": -23.084701538085938, |
| "step": 4380 |
| }, |
| { |
| "epoch": 2.0, |
| "learning_rate": 1.1060375443937086e-07, |
| "logits/chosen": -2.183781862258911, |
| "logits/rejected": -1.8434038162231445, |
| "logps/chosen": -84.59786224365234, |
| "logps/rejected": -118.7528076171875, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.4381895065307617, |
| "rewards/margins": 26.610469818115234, |
| "rewards/rejected": -24.17228126525879, |
| "step": 4390 |
| }, |
| { |
| "epoch": 2.01, |
| "learning_rate": 1.1009639776763063e-07, |
| "logits/chosen": -2.2394397258758545, |
| "logits/rejected": -1.8388830423355103, |
| "logps/chosen": -87.1556396484375, |
| "logps/rejected": -115.66267395019531, |
| "loss": 0.0024, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.9926955699920654, |
| "rewards/margins": 26.94419288635254, |
| "rewards/rejected": -23.951494216918945, |
| "step": 4400 |
| }, |
| { |
| "epoch": 2.01, |
| "eval_logits/chosen": -2.2128782272338867, |
| "eval_logits/rejected": -1.8610923290252686, |
| "eval_logps/chosen": -84.40758514404297, |
| "eval_logps/rejected": -115.89351654052734, |
| "eval_loss": 0.005497102625668049, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.211501359939575, |
| "eval_rewards/margins": 26.7427921295166, |
| "eval_rewards/rejected": -24.531293869018555, |
| "eval_runtime": 181.6374, |
| "eval_samples_per_second": 15.757, |
| "eval_steps_per_second": 0.985, |
| "step": 4400 |
| }, |
| { |
| "epoch": 2.01, |
| "learning_rate": 1.095890410958904e-07, |
| "logits/chosen": -2.1844065189361572, |
| "logits/rejected": -1.7922824621200562, |
| "logps/chosen": -87.32149505615234, |
| "logps/rejected": -124.73358154296875, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.739438533782959, |
| "rewards/margins": 29.961498260498047, |
| "rewards/rejected": -26.222061157226562, |
| "step": 4410 |
| }, |
| { |
| "epoch": 2.02, |
| "learning_rate": 1.0908168442415016e-07, |
| "logits/chosen": -2.1687610149383545, |
| "logits/rejected": -1.8179349899291992, |
| "logps/chosen": -82.60697937011719, |
| "logps/rejected": -117.3493881225586, |
| "loss": 0.0076, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0033984184265137, |
| "rewards/margins": 26.623882293701172, |
| "rewards/rejected": -24.620487213134766, |
| "step": 4420 |
| }, |
| { |
| "epoch": 2.02, |
| "learning_rate": 1.0857432775240993e-07, |
| "logits/chosen": -2.157742977142334, |
| "logits/rejected": -1.8218927383422852, |
| "logps/chosen": -81.67804718017578, |
| "logps/rejected": -123.06440734863281, |
| "loss": 0.0014, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.68721342086792, |
| "rewards/margins": 28.739349365234375, |
| "rewards/rejected": -26.052135467529297, |
| "step": 4430 |
| }, |
| { |
| "epoch": 2.03, |
| "learning_rate": 1.080669710806697e-07, |
| "logits/chosen": -2.275503396987915, |
| "logits/rejected": -1.8494739532470703, |
| "logps/chosen": -83.60503387451172, |
| "logps/rejected": -121.38664245605469, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1225333213806152, |
| "rewards/margins": 28.552631378173828, |
| "rewards/rejected": -25.430099487304688, |
| "step": 4440 |
| }, |
| { |
| "epoch": 2.03, |
| "learning_rate": 1.0755961440892946e-07, |
| "logits/chosen": -2.269984722137451, |
| "logits/rejected": -1.902269721031189, |
| "logps/chosen": -83.16432189941406, |
| "logps/rejected": -120.77657318115234, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8239471912384033, |
| "rewards/margins": 28.915185928344727, |
| "rewards/rejected": -26.091238021850586, |
| "step": 4450 |
| }, |
| { |
| "epoch": 2.04, |
| "learning_rate": 1.0705225773718923e-07, |
| "logits/chosen": -2.2269845008850098, |
| "logits/rejected": -1.8621666431427002, |
| "logps/chosen": -85.25446319580078, |
| "logps/rejected": -115.29423522949219, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4882960319519043, |
| "rewards/margins": 26.619338989257812, |
| "rewards/rejected": -24.13104248046875, |
| "step": 4460 |
| }, |
| { |
| "epoch": 2.04, |
| "learning_rate": 1.06544901065449e-07, |
| "logits/chosen": -2.236506700515747, |
| "logits/rejected": -1.9493324756622314, |
| "logps/chosen": -87.72699737548828, |
| "logps/rejected": -120.0291748046875, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.765307903289795, |
| "rewards/margins": 26.442195892333984, |
| "rewards/rejected": -23.676889419555664, |
| "step": 4470 |
| }, |
| { |
| "epoch": 2.04, |
| "learning_rate": 1.0603754439370876e-07, |
| "logits/chosen": -2.22512149810791, |
| "logits/rejected": -1.8360923528671265, |
| "logps/chosen": -85.01690673828125, |
| "logps/rejected": -123.11708068847656, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9220136404037476, |
| "rewards/margins": 28.249126434326172, |
| "rewards/rejected": -26.327117919921875, |
| "step": 4480 |
| }, |
| { |
| "epoch": 2.05, |
| "learning_rate": 1.0553018772196853e-07, |
| "logits/chosen": -2.271622896194458, |
| "logits/rejected": -1.8992735147476196, |
| "logps/chosen": -88.58778381347656, |
| "logps/rejected": -118.49006652832031, |
| "loss": 0.0011, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 3.1313204765319824, |
| "rewards/margins": 26.39713478088379, |
| "rewards/rejected": -23.265811920166016, |
| "step": 4490 |
| }, |
| { |
| "epoch": 2.05, |
| "learning_rate": 1.050228310502283e-07, |
| "logits/chosen": -2.2248878479003906, |
| "logits/rejected": -1.8648513555526733, |
| "logps/chosen": -84.03074645996094, |
| "logps/rejected": -119.88997650146484, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3918333053588867, |
| "rewards/margins": 26.75858497619629, |
| "rewards/rejected": -25.366750717163086, |
| "step": 4500 |
| }, |
| { |
| "epoch": 2.05, |
| "eval_logits/chosen": -2.2098562717437744, |
| "eval_logits/rejected": -1.8567416667938232, |
| "eval_logps/chosen": -84.76105499267578, |
| "eval_logps/rejected": -117.29058074951172, |
| "eval_loss": 0.005427930504083633, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 2.0347681045532227, |
| "eval_rewards/margins": 27.26459312438965, |
| "eval_rewards/rejected": -25.22982406616211, |
| "eval_runtime": 226.6245, |
| "eval_samples_per_second": 12.629, |
| "eval_steps_per_second": 0.79, |
| "step": 4500 |
| }, |
| { |
| "epoch": 2.06, |
| "learning_rate": 1.0451547437848806e-07, |
| "logits/chosen": -2.244274139404297, |
| "logits/rejected": -1.835097074508667, |
| "logps/chosen": -89.41958618164062, |
| "logps/rejected": -119.83707427978516, |
| "loss": 0.0031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5296854972839355, |
| "rewards/margins": 27.9240779876709, |
| "rewards/rejected": -25.394390106201172, |
| "step": 4510 |
| }, |
| { |
| "epoch": 2.06, |
| "learning_rate": 1.0400811770674783e-07, |
| "logits/chosen": -2.20641827583313, |
| "logits/rejected": -1.8095362186431885, |
| "logps/chosen": -79.14988708496094, |
| "logps/rejected": -120.5500717163086, |
| "loss": 0.0065, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.5764966011047363, |
| "rewards/margins": 29.600894927978516, |
| "rewards/rejected": -27.024398803710938, |
| "step": 4520 |
| }, |
| { |
| "epoch": 2.07, |
| "learning_rate": 1.035007610350076e-07, |
| "logits/chosen": -2.1728744506835938, |
| "logits/rejected": -1.750156044960022, |
| "logps/chosen": -90.58625793457031, |
| "logps/rejected": -115.90065002441406, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.2151265144348145, |
| "rewards/margins": 26.991992950439453, |
| "rewards/rejected": -24.776866912841797, |
| "step": 4530 |
| }, |
| { |
| "epoch": 2.07, |
| "learning_rate": 1.0299340436326736e-07, |
| "logits/chosen": -2.1972928047180176, |
| "logits/rejected": -1.8641561269760132, |
| "logps/chosen": -83.82379150390625, |
| "logps/rejected": -120.79838562011719, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7972400188446045, |
| "rewards/margins": 28.825088500976562, |
| "rewards/rejected": -26.027847290039062, |
| "step": 4540 |
| }, |
| { |
| "epoch": 2.08, |
| "learning_rate": 1.0248604769152713e-07, |
| "logits/chosen": -2.2066047191619873, |
| "logits/rejected": -1.8661673069000244, |
| "logps/chosen": -82.5276870727539, |
| "logps/rejected": -117.04951477050781, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4789516925811768, |
| "rewards/margins": 26.52178955078125, |
| "rewards/rejected": -24.042835235595703, |
| "step": 4550 |
| }, |
| { |
| "epoch": 2.08, |
| "learning_rate": 1.019786910197869e-07, |
| "logits/chosen": -2.2894678115844727, |
| "logits/rejected": -1.9717973470687866, |
| "logps/chosen": -85.42427825927734, |
| "logps/rejected": -122.86556243896484, |
| "loss": 0.0066, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.7783312797546387, |
| "rewards/margins": 26.929943084716797, |
| "rewards/rejected": -26.151615142822266, |
| "step": 4560 |
| }, |
| { |
| "epoch": 2.09, |
| "learning_rate": 1.0147133434804666e-07, |
| "logits/chosen": -2.2856619358062744, |
| "logits/rejected": -1.8935363292694092, |
| "logps/chosen": -83.85643005371094, |
| "logps/rejected": -116.0418930053711, |
| "loss": 0.0053, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.631080389022827, |
| "rewards/margins": 28.714336395263672, |
| "rewards/rejected": -26.083255767822266, |
| "step": 4570 |
| }, |
| { |
| "epoch": 2.09, |
| "learning_rate": 1.0096397767630643e-07, |
| "logits/chosen": -2.1582155227661133, |
| "logits/rejected": -1.7837406396865845, |
| "logps/chosen": -90.37853240966797, |
| "logps/rejected": -126.54129791259766, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.798124313354492, |
| "rewards/margins": 29.615198135375977, |
| "rewards/rejected": -25.817073822021484, |
| "step": 4580 |
| }, |
| { |
| "epoch": 2.1, |
| "learning_rate": 1.004566210045662e-07, |
| "logits/chosen": -2.217268705368042, |
| "logits/rejected": -1.9051685333251953, |
| "logps/chosen": -79.26915740966797, |
| "logps/rejected": -121.41270446777344, |
| "loss": 0.002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.768852949142456, |
| "rewards/margins": 27.323421478271484, |
| "rewards/rejected": -24.5545654296875, |
| "step": 4590 |
| }, |
| { |
| "epoch": 2.1, |
| "learning_rate": 9.994926433282596e-08, |
| "logits/chosen": -2.2435638904571533, |
| "logits/rejected": -1.8570373058319092, |
| "logps/chosen": -90.26069641113281, |
| "logps/rejected": -124.71513366699219, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2154905796051025, |
| "rewards/margins": 28.169509887695312, |
| "rewards/rejected": -25.954015731811523, |
| "step": 4600 |
| }, |
| { |
| "epoch": 2.1, |
| "eval_logits/chosen": -2.2148590087890625, |
| "eval_logits/rejected": -1.865024447441101, |
| "eval_logps/chosen": -84.85028839111328, |
| "eval_logps/rejected": -118.58063507080078, |
| "eval_loss": 0.005515058524906635, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.9901474714279175, |
| "eval_rewards/margins": 27.865001678466797, |
| "eval_rewards/rejected": -25.874853134155273, |
| "eval_runtime": 189.7173, |
| "eval_samples_per_second": 15.086, |
| "eval_steps_per_second": 0.944, |
| "step": 4600 |
| }, |
| { |
| "epoch": 2.1, |
| "learning_rate": 9.944190766108573e-08, |
| "logits/chosen": -2.2464184761047363, |
| "logits/rejected": -1.902503252029419, |
| "logps/chosen": -86.0322036743164, |
| "logps/rejected": -123.70219421386719, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7963413000106812, |
| "rewards/margins": 29.205463409423828, |
| "rewards/rejected": -27.409122467041016, |
| "step": 4610 |
| }, |
| { |
| "epoch": 2.11, |
| "learning_rate": 9.89345509893455e-08, |
| "logits/chosen": -2.134826183319092, |
| "logits/rejected": -1.8044805526733398, |
| "logps/chosen": -87.43550109863281, |
| "logps/rejected": -123.2356185913086, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1314964294433594, |
| "rewards/margins": 29.698129653930664, |
| "rewards/rejected": -26.566635131835938, |
| "step": 4620 |
| }, |
| { |
| "epoch": 2.11, |
| "learning_rate": 9.842719431760526e-08, |
| "logits/chosen": -2.146523952484131, |
| "logits/rejected": -1.7418187856674194, |
| "logps/chosen": -83.50428771972656, |
| "logps/rejected": -122.54347229003906, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.376321792602539, |
| "rewards/margins": 30.02010726928711, |
| "rewards/rejected": -27.643783569335938, |
| "step": 4630 |
| }, |
| { |
| "epoch": 2.12, |
| "learning_rate": 9.791983764586503e-08, |
| "logits/chosen": -2.262716770172119, |
| "logits/rejected": -1.8752014636993408, |
| "logps/chosen": -84.97921752929688, |
| "logps/rejected": -122.33912658691406, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.812735915184021, |
| "rewards/margins": 29.280685424804688, |
| "rewards/rejected": -27.46795082092285, |
| "step": 4640 |
| }, |
| { |
| "epoch": 2.12, |
| "learning_rate": 9.74124809741248e-08, |
| "logits/chosen": -2.2482247352600098, |
| "logits/rejected": -1.9075597524642944, |
| "logps/chosen": -81.56404113769531, |
| "logps/rejected": -120.65312194824219, |
| "loss": 0.0056, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.012248992919922, |
| "rewards/margins": 28.712228775024414, |
| "rewards/rejected": -26.699981689453125, |
| "step": 4650 |
| }, |
| { |
| "epoch": 2.13, |
| "learning_rate": 9.690512430238456e-08, |
| "logits/chosen": -2.1412789821624756, |
| "logits/rejected": -1.8419468402862549, |
| "logps/chosen": -83.85114288330078, |
| "logps/rejected": -121.31649017333984, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0328783988952637, |
| "rewards/margins": 27.409412384033203, |
| "rewards/rejected": -26.376529693603516, |
| "step": 4660 |
| }, |
| { |
| "epoch": 2.13, |
| "learning_rate": 9.639776763064433e-08, |
| "logits/chosen": -2.2971882820129395, |
| "logits/rejected": -1.9334943294525146, |
| "logps/chosen": -82.55015563964844, |
| "logps/rejected": -119.16120910644531, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0977225303649902, |
| "rewards/margins": 28.73293113708496, |
| "rewards/rejected": -26.635211944580078, |
| "step": 4670 |
| }, |
| { |
| "epoch": 2.14, |
| "learning_rate": 9.58904109589041e-08, |
| "logits/chosen": -2.325941562652588, |
| "logits/rejected": -1.964868187904358, |
| "logps/chosen": -87.65157318115234, |
| "logps/rejected": -125.1788330078125, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.739508032798767, |
| "rewards/margins": 30.17959213256836, |
| "rewards/rejected": -28.44008445739746, |
| "step": 4680 |
| }, |
| { |
| "epoch": 2.14, |
| "learning_rate": 9.538305428716386e-08, |
| "logits/chosen": -2.1548006534576416, |
| "logits/rejected": -1.8352489471435547, |
| "logps/chosen": -83.15804290771484, |
| "logps/rejected": -114.95072174072266, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4212478399276733, |
| "rewards/margins": 27.054697036743164, |
| "rewards/rejected": -25.63344955444336, |
| "step": 4690 |
| }, |
| { |
| "epoch": 2.15, |
| "learning_rate": 9.487569761542363e-08, |
| "logits/chosen": -2.263066530227661, |
| "logits/rejected": -1.8140977621078491, |
| "logps/chosen": -93.13688659667969, |
| "logps/rejected": -124.15168762207031, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.275789737701416, |
| "rewards/margins": 29.475337982177734, |
| "rewards/rejected": -26.199548721313477, |
| "step": 4700 |
| }, |
| { |
| "epoch": 2.15, |
| "eval_logits/chosen": -2.2139129638671875, |
| "eval_logits/rejected": -1.8623522520065308, |
| "eval_logps/chosen": -85.51490783691406, |
| "eval_logps/rejected": -120.75127410888672, |
| "eval_loss": 0.005448976997286081, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.657840371131897, |
| "eval_rewards/margins": 28.618017196655273, |
| "eval_rewards/rejected": -26.960174560546875, |
| "eval_runtime": 230.7918, |
| "eval_samples_per_second": 12.401, |
| "eval_steps_per_second": 0.776, |
| "step": 4700 |
| }, |
| { |
| "epoch": 2.15, |
| "learning_rate": 9.43683409436834e-08, |
| "logits/chosen": -2.2211930751800537, |
| "logits/rejected": -1.881513237953186, |
| "logps/chosen": -82.05587005615234, |
| "logps/rejected": -126.84517669677734, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6722524166107178, |
| "rewards/margins": 30.808147430419922, |
| "rewards/rejected": -28.13589859008789, |
| "step": 4710 |
| }, |
| { |
| "epoch": 2.15, |
| "learning_rate": 9.386098427194316e-08, |
| "logits/chosen": -2.1461105346679688, |
| "logits/rejected": -1.7770576477050781, |
| "logps/chosen": -85.20562744140625, |
| "logps/rejected": -125.3041763305664, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.119962692260742, |
| "rewards/margins": 29.933517456054688, |
| "rewards/rejected": -27.813552856445312, |
| "step": 4720 |
| }, |
| { |
| "epoch": 2.16, |
| "learning_rate": 9.335362760020293e-08, |
| "logits/chosen": -2.215076208114624, |
| "logits/rejected": -1.8217270374298096, |
| "logps/chosen": -84.71824645996094, |
| "logps/rejected": -120.50927734375, |
| "loss": 0.0058, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.6982955932617188, |
| "rewards/margins": 29.129375457763672, |
| "rewards/rejected": -26.431079864501953, |
| "step": 4730 |
| }, |
| { |
| "epoch": 2.16, |
| "learning_rate": 9.28462709284627e-08, |
| "logits/chosen": -2.2664477825164795, |
| "logits/rejected": -1.8872871398925781, |
| "logps/chosen": -86.96624755859375, |
| "logps/rejected": -124.43575286865234, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4599993228912354, |
| "rewards/margins": 30.88054847717285, |
| "rewards/rejected": -29.420547485351562, |
| "step": 4740 |
| }, |
| { |
| "epoch": 2.17, |
| "learning_rate": 9.233891425672246e-08, |
| "logits/chosen": -2.2344231605529785, |
| "logits/rejected": -1.7941217422485352, |
| "logps/chosen": -88.73806762695312, |
| "logps/rejected": -124.6733627319336, |
| "loss": 0.0054, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4499599933624268, |
| "rewards/margins": 30.964941024780273, |
| "rewards/rejected": -27.51498031616211, |
| "step": 4750 |
| }, |
| { |
| "epoch": 2.17, |
| "learning_rate": 9.183155758498223e-08, |
| "logits/chosen": -2.26192045211792, |
| "logits/rejected": -1.9525654315948486, |
| "logps/chosen": -86.94519805908203, |
| "logps/rejected": -128.98985290527344, |
| "loss": 0.0012, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.0427141189575195, |
| "rewards/margins": 29.724853515625, |
| "rewards/rejected": -28.682140350341797, |
| "step": 4760 |
| }, |
| { |
| "epoch": 2.18, |
| "learning_rate": 9.1324200913242e-08, |
| "logits/chosen": -2.2315638065338135, |
| "logits/rejected": -1.844601035118103, |
| "logps/chosen": -90.42842864990234, |
| "logps/rejected": -123.21382904052734, |
| "loss": 0.0019, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7231295108795166, |
| "rewards/margins": 28.74625587463379, |
| "rewards/rejected": -27.023128509521484, |
| "step": 4770 |
| }, |
| { |
| "epoch": 2.18, |
| "learning_rate": 9.081684424150176e-08, |
| "logits/chosen": -2.2464542388916016, |
| "logits/rejected": -1.9291092157363892, |
| "logps/chosen": -90.54940032958984, |
| "logps/rejected": -130.29818725585938, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4829115867614746, |
| "rewards/margins": 30.492401123046875, |
| "rewards/rejected": -29.00948715209961, |
| "step": 4780 |
| }, |
| { |
| "epoch": 2.19, |
| "learning_rate": 9.030948756976153e-08, |
| "logits/chosen": -2.249579906463623, |
| "logits/rejected": -1.7951542139053345, |
| "logps/chosen": -84.71315002441406, |
| "logps/rejected": -123.6868896484375, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.647388458251953, |
| "rewards/margins": 31.460529327392578, |
| "rewards/rejected": -28.813140869140625, |
| "step": 4790 |
| }, |
| { |
| "epoch": 2.19, |
| "learning_rate": 8.98021308980213e-08, |
| "logits/chosen": -2.2210094928741455, |
| "logits/rejected": -1.8732092380523682, |
| "logps/chosen": -83.642578125, |
| "logps/rejected": -124.18983459472656, |
| "loss": 0.0064, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.0244059562683105, |
| "rewards/margins": 29.753732681274414, |
| "rewards/rejected": -27.729320526123047, |
| "step": 4800 |
| }, |
| { |
| "epoch": 2.19, |
| "eval_logits/chosen": -2.208674192428589, |
| "eval_logits/rejected": -1.8558579683303833, |
| "eval_logps/chosen": -86.5013198852539, |
| "eval_logps/rejected": -123.4262466430664, |
| "eval_loss": 0.005769502837210894, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.1646300554275513, |
| "eval_rewards/margins": 29.462299346923828, |
| "eval_rewards/rejected": -28.29766845703125, |
| "eval_runtime": 258.0094, |
| "eval_samples_per_second": 11.093, |
| "eval_steps_per_second": 0.694, |
| "step": 4800 |
| }, |
| { |
| "epoch": 2.2, |
| "learning_rate": 8.929477422628106e-08, |
| "logits/chosen": -2.2377803325653076, |
| "logits/rejected": -1.8689501285552979, |
| "logps/chosen": -88.69053649902344, |
| "logps/rejected": -133.93704223632812, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5955907106399536, |
| "rewards/margins": 29.29754066467285, |
| "rewards/rejected": -27.701946258544922, |
| "step": 4810 |
| }, |
| { |
| "epoch": 2.2, |
| "learning_rate": 8.878741755454083e-08, |
| "logits/chosen": -2.179999589920044, |
| "logits/rejected": -1.8029935359954834, |
| "logps/chosen": -90.18666076660156, |
| "logps/rejected": -128.03399658203125, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3174841403961182, |
| "rewards/margins": 30.12929344177246, |
| "rewards/rejected": -28.811809539794922, |
| "step": 4820 |
| }, |
| { |
| "epoch": 2.2, |
| "learning_rate": 8.82800608828006e-08, |
| "logits/chosen": -2.2542881965637207, |
| "logits/rejected": -1.860487699508667, |
| "logps/chosen": -84.78041076660156, |
| "logps/rejected": -127.14323425292969, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2697548866271973, |
| "rewards/margins": 31.854806900024414, |
| "rewards/rejected": -29.585052490234375, |
| "step": 4830 |
| }, |
| { |
| "epoch": 2.21, |
| "learning_rate": 8.777270421106036e-08, |
| "logits/chosen": -2.2306056022644043, |
| "logits/rejected": -1.9451026916503906, |
| "logps/chosen": -84.75010681152344, |
| "logps/rejected": -127.5153579711914, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.20773085951805115, |
| "rewards/margins": 29.199344635009766, |
| "rewards/rejected": -28.99161720275879, |
| "step": 4840 |
| }, |
| { |
| "epoch": 2.21, |
| "learning_rate": 8.726534753932013e-08, |
| "logits/chosen": -2.195384979248047, |
| "logits/rejected": -1.8609682321548462, |
| "logps/chosen": -84.73804473876953, |
| "logps/rejected": -123.91796875, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5444185733795166, |
| "rewards/margins": 30.432912826538086, |
| "rewards/rejected": -28.88849449157715, |
| "step": 4850 |
| }, |
| { |
| "epoch": 2.22, |
| "learning_rate": 8.67579908675799e-08, |
| "logits/chosen": -2.1483521461486816, |
| "logits/rejected": -1.7701698541641235, |
| "logps/chosen": -81.2613754272461, |
| "logps/rejected": -121.82652282714844, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8260562419891357, |
| "rewards/margins": 30.310237884521484, |
| "rewards/rejected": -27.484180450439453, |
| "step": 4860 |
| }, |
| { |
| "epoch": 2.22, |
| "learning_rate": 8.625063419583966e-08, |
| "logits/chosen": -2.243568181991577, |
| "logits/rejected": -1.941277265548706, |
| "logps/chosen": -84.82609558105469, |
| "logps/rejected": -127.32414245605469, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3905398845672607, |
| "rewards/margins": 30.30340003967285, |
| "rewards/rejected": -28.912860870361328, |
| "step": 4870 |
| }, |
| { |
| "epoch": 2.23, |
| "learning_rate": 8.574327752409943e-08, |
| "logits/chosen": -2.1722254753112793, |
| "logits/rejected": -1.8854080438613892, |
| "logps/chosen": -83.84065246582031, |
| "logps/rejected": -128.77059936523438, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0625784397125244, |
| "rewards/margins": 31.2793025970459, |
| "rewards/rejected": -30.216724395751953, |
| "step": 4880 |
| }, |
| { |
| "epoch": 2.23, |
| "learning_rate": 8.52359208523592e-08, |
| "logits/chosen": -2.1614387035369873, |
| "logits/rejected": -1.8331111669540405, |
| "logps/chosen": -85.25975036621094, |
| "logps/rejected": -130.95721435546875, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8772072792053223, |
| "rewards/margins": 31.198410034179688, |
| "rewards/rejected": -28.32120132446289, |
| "step": 4890 |
| }, |
| { |
| "epoch": 2.24, |
| "learning_rate": 8.472856418061896e-08, |
| "logits/chosen": -2.2037620544433594, |
| "logits/rejected": -1.8145354986190796, |
| "logps/chosen": -86.72196197509766, |
| "logps/rejected": -121.60040283203125, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.486250877380371, |
| "rewards/margins": 30.308456420898438, |
| "rewards/rejected": -27.822208404541016, |
| "step": 4900 |
| }, |
| { |
| "epoch": 2.24, |
| "eval_logits/chosen": -2.2160322666168213, |
| "eval_logits/rejected": -1.868016242980957, |
| "eval_logps/chosen": -86.11427307128906, |
| "eval_logps/rejected": -123.58334350585938, |
| "eval_loss": 0.0056231957860291, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.3581570386886597, |
| "eval_rewards/margins": 29.7343692779541, |
| "eval_rewards/rejected": -28.376211166381836, |
| "eval_runtime": 178.9426, |
| "eval_samples_per_second": 15.994, |
| "eval_steps_per_second": 1.0, |
| "step": 4900 |
| }, |
| { |
| "epoch": 2.24, |
| "learning_rate": 8.422120750887873e-08, |
| "logits/chosen": -2.2726593017578125, |
| "logits/rejected": -1.8736642599105835, |
| "logps/chosen": -87.76054382324219, |
| "logps/rejected": -124.5219955444336, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.735135316848755, |
| "rewards/margins": 30.699026107788086, |
| "rewards/rejected": -27.963891983032227, |
| "step": 4910 |
| }, |
| { |
| "epoch": 2.25, |
| "learning_rate": 8.37138508371385e-08, |
| "logits/chosen": -2.2303450107574463, |
| "logits/rejected": -1.8574903011322021, |
| "logps/chosen": -87.68033599853516, |
| "logps/rejected": -125.37784576416016, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6258609294891357, |
| "rewards/margins": 30.00998878479004, |
| "rewards/rejected": -28.384124755859375, |
| "step": 4920 |
| }, |
| { |
| "epoch": 2.25, |
| "learning_rate": 8.320649416539826e-08, |
| "logits/chosen": -2.1431241035461426, |
| "logits/rejected": -1.8002105951309204, |
| "logps/chosen": -86.46646881103516, |
| "logps/rejected": -127.82108306884766, |
| "loss": 0.0034, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.7209049463272095, |
| "rewards/margins": 30.69203758239746, |
| "rewards/rejected": -28.971134185791016, |
| "step": 4930 |
| }, |
| { |
| "epoch": 2.25, |
| "learning_rate": 8.269913749365803e-08, |
| "logits/chosen": -2.21543550491333, |
| "logits/rejected": -1.858415961265564, |
| "logps/chosen": -85.4444808959961, |
| "logps/rejected": -125.85990142822266, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.123652219772339, |
| "rewards/margins": 29.99489974975586, |
| "rewards/rejected": -27.87125015258789, |
| "step": 4940 |
| }, |
| { |
| "epoch": 2.26, |
| "learning_rate": 8.21917808219178e-08, |
| "logits/chosen": -2.226576328277588, |
| "logits/rejected": -1.8361542224884033, |
| "logps/chosen": -88.27388000488281, |
| "logps/rejected": -124.52458190917969, |
| "loss": 0.0054, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.222522020339966, |
| "rewards/margins": 29.900218963623047, |
| "rewards/rejected": -27.677698135375977, |
| "step": 4950 |
| }, |
| { |
| "epoch": 2.26, |
| "learning_rate": 8.168442415017756e-08, |
| "logits/chosen": -2.211759567260742, |
| "logits/rejected": -1.8759946823120117, |
| "logps/chosen": -91.97505187988281, |
| "logps/rejected": -127.7925796508789, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.8236006498336792, |
| "rewards/margins": 30.634103775024414, |
| "rewards/rejected": -29.810501098632812, |
| "step": 4960 |
| }, |
| { |
| "epoch": 2.27, |
| "learning_rate": 8.117706747843733e-08, |
| "logits/chosen": -2.220738172531128, |
| "logits/rejected": -1.8724660873413086, |
| "logps/chosen": -87.83768463134766, |
| "logps/rejected": -127.0542984008789, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5908282995224, |
| "rewards/margins": 30.813274383544922, |
| "rewards/rejected": -29.22244644165039, |
| "step": 4970 |
| }, |
| { |
| "epoch": 2.27, |
| "learning_rate": 8.06697108066971e-08, |
| "logits/chosen": -2.2144250869750977, |
| "logits/rejected": -1.8192403316497803, |
| "logps/chosen": -88.7645034790039, |
| "logps/rejected": -123.9654769897461, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7449915409088135, |
| "rewards/margins": 30.582874298095703, |
| "rewards/rejected": -28.8378849029541, |
| "step": 4980 |
| }, |
| { |
| "epoch": 2.28, |
| "learning_rate": 8.016235413495687e-08, |
| "logits/chosen": -2.1595869064331055, |
| "logits/rejected": -1.8579118251800537, |
| "logps/chosen": -81.31648254394531, |
| "logps/rejected": -125.78511810302734, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1897120475769043, |
| "rewards/margins": 31.16072654724121, |
| "rewards/rejected": -28.97101402282715, |
| "step": 4990 |
| }, |
| { |
| "epoch": 2.28, |
| "learning_rate": 7.965499746321664e-08, |
| "logits/chosen": -2.163888692855835, |
| "logits/rejected": -1.7756332159042358, |
| "logps/chosen": -91.29290771484375, |
| "logps/rejected": -126.38360595703125, |
| "loss": 0.0025, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.3966686725616455, |
| "rewards/margins": 29.905893325805664, |
| "rewards/rejected": -27.509225845336914, |
| "step": 5000 |
| }, |
| { |
| "epoch": 2.28, |
| "eval_logits/chosen": -2.2226171493530273, |
| "eval_logits/rejected": -1.8767516613006592, |
| "eval_logps/chosen": -86.72997283935547, |
| "eval_logps/rejected": -125.4724349975586, |
| "eval_loss": 0.005568630062043667, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.0503071546554565, |
| "eval_rewards/margins": 30.37106704711914, |
| "eval_rewards/rejected": -29.320756912231445, |
| "eval_runtime": 265.4131, |
| "eval_samples_per_second": 10.783, |
| "eval_steps_per_second": 0.674, |
| "step": 5000 |
| }, |
| { |
| "epoch": 2.29, |
| "learning_rate": 7.91476407914764e-08, |
| "logits/chosen": -2.237316608428955, |
| "logits/rejected": -1.8469164371490479, |
| "logps/chosen": -86.86993408203125, |
| "logps/rejected": -128.10240173339844, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2446770668029785, |
| "rewards/margins": 31.02500343322754, |
| "rewards/rejected": -29.78032875061035, |
| "step": 5010 |
| }, |
| { |
| "epoch": 2.29, |
| "learning_rate": 7.864028411973617e-08, |
| "logits/chosen": -2.212674379348755, |
| "logits/rejected": -1.7991310358047485, |
| "logps/chosen": -90.48421478271484, |
| "logps/rejected": -128.47349548339844, |
| "loss": 0.0067, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.104187488555908, |
| "rewards/margins": 31.993030548095703, |
| "rewards/rejected": -29.888843536376953, |
| "step": 5020 |
| }, |
| { |
| "epoch": 2.3, |
| "learning_rate": 7.813292744799594e-08, |
| "logits/chosen": -2.192707061767578, |
| "logits/rejected": -1.812227487564087, |
| "logps/chosen": -85.58504486083984, |
| "logps/rejected": -125.02815246582031, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6018452644348145, |
| "rewards/margins": 31.598779678344727, |
| "rewards/rejected": -28.996929168701172, |
| "step": 5030 |
| }, |
| { |
| "epoch": 2.3, |
| "learning_rate": 7.76255707762557e-08, |
| "logits/chosen": -2.2332510948181152, |
| "logits/rejected": -1.9000991582870483, |
| "logps/chosen": -87.93112182617188, |
| "logps/rejected": -124.27226257324219, |
| "loss": 0.001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8778412938117981, |
| "rewards/margins": 29.009052276611328, |
| "rewards/rejected": -28.131210327148438, |
| "step": 5040 |
| }, |
| { |
| "epoch": 2.31, |
| "learning_rate": 7.711821410451547e-08, |
| "logits/chosen": -2.2971322536468506, |
| "logits/rejected": -1.955288290977478, |
| "logps/chosen": -84.05998229980469, |
| "logps/rejected": -121.64460754394531, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0523746013641357, |
| "rewards/margins": 30.503372192382812, |
| "rewards/rejected": -28.45099449157715, |
| "step": 5050 |
| }, |
| { |
| "epoch": 2.31, |
| "learning_rate": 7.661085743277524e-08, |
| "logits/chosen": -2.2180066108703613, |
| "logits/rejected": -1.8513801097869873, |
| "logps/chosen": -91.88666534423828, |
| "logps/rejected": -131.45933532714844, |
| "loss": 0.0019, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.476855516433716, |
| "rewards/margins": 32.50453186035156, |
| "rewards/rejected": -30.02767562866211, |
| "step": 5060 |
| }, |
| { |
| "epoch": 2.31, |
| "learning_rate": 7.6103500761035e-08, |
| "logits/chosen": -2.202540636062622, |
| "logits/rejected": -1.883195161819458, |
| "logps/chosen": -86.0487060546875, |
| "logps/rejected": -127.7849349975586, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5693166255950928, |
| "rewards/margins": 30.323623657226562, |
| "rewards/rejected": -29.75430679321289, |
| "step": 5070 |
| }, |
| { |
| "epoch": 2.32, |
| "learning_rate": 7.559614408929477e-08, |
| "logits/chosen": -2.167088031768799, |
| "logits/rejected": -1.7546894550323486, |
| "logps/chosen": -86.92243194580078, |
| "logps/rejected": -121.72308349609375, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5353543758392334, |
| "rewards/margins": 30.667156219482422, |
| "rewards/rejected": -28.13180160522461, |
| "step": 5080 |
| }, |
| { |
| "epoch": 2.32, |
| "learning_rate": 7.508878741755454e-08, |
| "logits/chosen": -2.1240665912628174, |
| "logits/rejected": -1.8034683465957642, |
| "logps/chosen": -86.10752868652344, |
| "logps/rejected": -130.32803344726562, |
| "loss": 0.0055, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.6282140016555786, |
| "rewards/margins": 30.408313751220703, |
| "rewards/rejected": -29.780099868774414, |
| "step": 5090 |
| }, |
| { |
| "epoch": 2.33, |
| "learning_rate": 7.45814307458143e-08, |
| "logits/chosen": -2.234485387802124, |
| "logits/rejected": -1.8363311290740967, |
| "logps/chosen": -89.19425964355469, |
| "logps/rejected": -129.45681762695312, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9281721115112305, |
| "rewards/margins": 33.132511138916016, |
| "rewards/rejected": -31.204341888427734, |
| "step": 5100 |
| }, |
| { |
| "epoch": 2.33, |
| "eval_logits/chosen": -2.2230384349823, |
| "eval_logits/rejected": -1.8779499530792236, |
| "eval_logps/chosen": -86.5483627319336, |
| "eval_logps/rejected": -125.72158813476562, |
| "eval_loss": 0.005534125491976738, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 1.1411113739013672, |
| "eval_rewards/margins": 30.586444854736328, |
| "eval_rewards/rejected": -29.44533348083496, |
| "eval_runtime": 203.014, |
| "eval_samples_per_second": 14.098, |
| "eval_steps_per_second": 0.882, |
| "step": 5100 |
| }, |
| { |
| "epoch": 2.33, |
| "learning_rate": 7.407407407407407e-08, |
| "logits/chosen": -2.23368239402771, |
| "logits/rejected": -1.8570277690887451, |
| "logps/chosen": -87.19905853271484, |
| "logps/rejected": -127.96573638916016, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1923465728759766, |
| "rewards/margins": 31.500961303710938, |
| "rewards/rejected": -30.30861473083496, |
| "step": 5110 |
| }, |
| { |
| "epoch": 2.34, |
| "learning_rate": 7.356671740233384e-08, |
| "logits/chosen": -2.14494252204895, |
| "logits/rejected": -1.8105385303497314, |
| "logps/chosen": -83.87150573730469, |
| "logps/rejected": -126.78532409667969, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.35639578104019165, |
| "rewards/margins": 30.178089141845703, |
| "rewards/rejected": -29.82169532775879, |
| "step": 5120 |
| }, |
| { |
| "epoch": 2.34, |
| "learning_rate": 7.30593607305936e-08, |
| "logits/chosen": -2.253058671951294, |
| "logits/rejected": -1.9436228275299072, |
| "logps/chosen": -90.07007598876953, |
| "logps/rejected": -127.9438705444336, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.44818297028541565, |
| "rewards/margins": 30.035837173461914, |
| "rewards/rejected": -30.4840145111084, |
| "step": 5130 |
| }, |
| { |
| "epoch": 2.35, |
| "learning_rate": 7.255200405885337e-08, |
| "logits/chosen": -2.2380034923553467, |
| "logits/rejected": -1.849329948425293, |
| "logps/chosen": -91.55721282958984, |
| "logps/rejected": -132.04197692871094, |
| "loss": 0.001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7558342218399048, |
| "rewards/margins": 31.776962280273438, |
| "rewards/rejected": -30.021127700805664, |
| "step": 5140 |
| }, |
| { |
| "epoch": 2.35, |
| "learning_rate": 7.204464738711314e-08, |
| "logits/chosen": -2.249427318572998, |
| "logits/rejected": -1.916865348815918, |
| "logps/chosen": -84.08346557617188, |
| "logps/rejected": -127.4309310913086, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0999155044555664, |
| "rewards/margins": 31.15311622619629, |
| "rewards/rejected": -30.053197860717773, |
| "step": 5150 |
| }, |
| { |
| "epoch": 2.36, |
| "learning_rate": 7.15372907153729e-08, |
| "logits/chosen": -2.164412498474121, |
| "logits/rejected": -1.8164761066436768, |
| "logps/chosen": -84.87574768066406, |
| "logps/rejected": -127.91072845458984, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4683095216751099, |
| "rewards/margins": 30.6422176361084, |
| "rewards/rejected": -29.173908233642578, |
| "step": 5160 |
| }, |
| { |
| "epoch": 2.36, |
| "learning_rate": 7.102993404363267e-08, |
| "logits/chosen": -2.2733216285705566, |
| "logits/rejected": -1.8871898651123047, |
| "logps/chosen": -84.68191528320312, |
| "logps/rejected": -128.5628204345703, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4870762825012207, |
| "rewards/margins": 31.898874282836914, |
| "rewards/rejected": -30.41179847717285, |
| "step": 5170 |
| }, |
| { |
| "epoch": 2.36, |
| "learning_rate": 7.052257737189244e-08, |
| "logits/chosen": -2.1040546894073486, |
| "logits/rejected": -1.794029951095581, |
| "logps/chosen": -87.33931732177734, |
| "logps/rejected": -122.92645263671875, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.523239016532898, |
| "rewards/margins": 28.69746971130371, |
| "rewards/rejected": -28.174230575561523, |
| "step": 5180 |
| }, |
| { |
| "epoch": 2.37, |
| "learning_rate": 7.00152207001522e-08, |
| "logits/chosen": -2.2180774211883545, |
| "logits/rejected": -1.8783124685287476, |
| "logps/chosen": -88.9879150390625, |
| "logps/rejected": -129.85577392578125, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.285766363143921, |
| "rewards/margins": 31.67111587524414, |
| "rewards/rejected": -30.38534927368164, |
| "step": 5190 |
| }, |
| { |
| "epoch": 2.37, |
| "learning_rate": 6.950786402841197e-08, |
| "logits/chosen": -2.2432150840759277, |
| "logits/rejected": -1.8757518529891968, |
| "logps/chosen": -96.52727508544922, |
| "logps/rejected": -133.89486694335938, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.2539239525794983, |
| "rewards/margins": 30.236125946044922, |
| "rewards/rejected": -30.49005126953125, |
| "step": 5200 |
| }, |
| { |
| "epoch": 2.37, |
| "eval_logits/chosen": -2.2150111198425293, |
| "eval_logits/rejected": -1.8662109375, |
| "eval_logps/chosen": -86.7686767578125, |
| "eval_logps/rejected": -126.29188537597656, |
| "eval_loss": 0.005586822517216206, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 1.030951976776123, |
| "eval_rewards/margins": 30.761432647705078, |
| "eval_rewards/rejected": -29.730480194091797, |
| "eval_runtime": 372.2708, |
| "eval_samples_per_second": 7.688, |
| "eval_steps_per_second": 0.481, |
| "step": 5200 |
| }, |
| { |
| "epoch": 2.38, |
| "learning_rate": 6.900050735667174e-08, |
| "logits/chosen": -2.2496633529663086, |
| "logits/rejected": -1.9013668298721313, |
| "logps/chosen": -85.10989379882812, |
| "logps/rejected": -126.21388244628906, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0123857259750366, |
| "rewards/margins": 28.938159942626953, |
| "rewards/rejected": -27.925771713256836, |
| "step": 5210 |
| }, |
| { |
| "epoch": 2.38, |
| "learning_rate": 6.84931506849315e-08, |
| "logits/chosen": -2.169320583343506, |
| "logits/rejected": -1.8793160915374756, |
| "logps/chosen": -83.18421173095703, |
| "logps/rejected": -132.3877410888672, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.122344732284546, |
| "rewards/margins": 30.82855796813965, |
| "rewards/rejected": -29.70621109008789, |
| "step": 5220 |
| }, |
| { |
| "epoch": 2.39, |
| "learning_rate": 6.798579401319127e-08, |
| "logits/chosen": -2.3072543144226074, |
| "logits/rejected": -1.8543882369995117, |
| "logps/chosen": -92.82881164550781, |
| "logps/rejected": -127.92374420166016, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.4654512405395508, |
| "rewards/margins": 31.590072631835938, |
| "rewards/rejected": -30.124622344970703, |
| "step": 5230 |
| }, |
| { |
| "epoch": 2.39, |
| "learning_rate": 6.747843734145104e-08, |
| "logits/chosen": -2.2160370349884033, |
| "logits/rejected": -1.8520358800888062, |
| "logps/chosen": -87.04851531982422, |
| "logps/rejected": -126.8850326538086, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.426818370819092, |
| "rewards/margins": 32.05809020996094, |
| "rewards/rejected": -29.631271362304688, |
| "step": 5240 |
| }, |
| { |
| "epoch": 2.4, |
| "learning_rate": 6.69710806697108e-08, |
| "logits/chosen": -2.2358498573303223, |
| "logits/rejected": -1.92721426486969, |
| "logps/chosen": -83.96819305419922, |
| "logps/rejected": -127.56068420410156, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 1.6845203638076782, |
| "rewards/margins": 31.140390396118164, |
| "rewards/rejected": -29.455867767333984, |
| "step": 5250 |
| }, |
| { |
| "epoch": 2.4, |
| "learning_rate": 6.646372399797057e-08, |
| "logits/chosen": -2.224752426147461, |
| "logits/rejected": -1.8739010095596313, |
| "logps/chosen": -83.2956771850586, |
| "logps/rejected": -126.33967590332031, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8728569746017456, |
| "rewards/margins": 29.507293701171875, |
| "rewards/rejected": -28.63443374633789, |
| "step": 5260 |
| }, |
| { |
| "epoch": 2.41, |
| "learning_rate": 6.595636732623034e-08, |
| "logits/chosen": -2.2283332347869873, |
| "logits/rejected": -1.898648977279663, |
| "logps/chosen": -89.44114685058594, |
| "logps/rejected": -134.07838439941406, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2760619521141052, |
| "rewards/margins": 31.4327335357666, |
| "rewards/rejected": -31.156673431396484, |
| "step": 5270 |
| }, |
| { |
| "epoch": 2.41, |
| "learning_rate": 6.54490106544901e-08, |
| "logits/chosen": -2.217665195465088, |
| "logits/rejected": -1.7950804233551025, |
| "logps/chosen": -91.71220397949219, |
| "logps/rejected": -131.68295288085938, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5391443967819214, |
| "rewards/margins": 33.635643005371094, |
| "rewards/rejected": -32.09649658203125, |
| "step": 5280 |
| }, |
| { |
| "epoch": 2.41, |
| "learning_rate": 6.494165398274987e-08, |
| "logits/chosen": -2.1419761180877686, |
| "logits/rejected": -1.7783762216567993, |
| "logps/chosen": -90.9324722290039, |
| "logps/rejected": -132.05467224121094, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3793599605560303, |
| "rewards/margins": 31.754810333251953, |
| "rewards/rejected": -30.375452041625977, |
| "step": 5290 |
| }, |
| { |
| "epoch": 2.42, |
| "learning_rate": 6.443429731100964e-08, |
| "logits/chosen": -2.23848032951355, |
| "logits/rejected": -1.9507777690887451, |
| "logps/chosen": -83.6585464477539, |
| "logps/rejected": -128.34262084960938, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.07423808425664902, |
| "rewards/margins": 31.000595092773438, |
| "rewards/rejected": -30.926355361938477, |
| "step": 5300 |
| }, |
| { |
| "epoch": 2.42, |
| "eval_logits/chosen": -2.220287322998047, |
| "eval_logits/rejected": -1.8729933500289917, |
| "eval_logps/chosen": -87.28707885742188, |
| "eval_logps/rejected": -127.75875091552734, |
| "eval_loss": 0.005639108829200268, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.7717540264129639, |
| "eval_rewards/margins": 31.235660552978516, |
| "eval_rewards/rejected": -30.463911056518555, |
| "eval_runtime": 204.4438, |
| "eval_samples_per_second": 13.999, |
| "eval_steps_per_second": 0.876, |
| "step": 5300 |
| }, |
| { |
| "epoch": 2.42, |
| "learning_rate": 6.39269406392694e-08, |
| "logits/chosen": -2.217376232147217, |
| "logits/rejected": -1.8636280298233032, |
| "logps/chosen": -85.64130401611328, |
| "logps/rejected": -130.76148986816406, |
| "loss": 0.0045, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.24725079536438, |
| "rewards/margins": 32.80742263793945, |
| "rewards/rejected": -30.5601749420166, |
| "step": 5310 |
| }, |
| { |
| "epoch": 2.43, |
| "learning_rate": 6.341958396752917e-08, |
| "logits/chosen": -2.2781529426574707, |
| "logits/rejected": -1.9017364978790283, |
| "logps/chosen": -85.34228515625, |
| "logps/rejected": -126.925537109375, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1655864715576172, |
| "rewards/margins": 31.301239013671875, |
| "rewards/rejected": -30.135656356811523, |
| "step": 5320 |
| }, |
| { |
| "epoch": 2.43, |
| "learning_rate": 6.291222729578894e-08, |
| "logits/chosen": -2.2987561225891113, |
| "logits/rejected": -1.894020438194275, |
| "logps/chosen": -90.0018539428711, |
| "logps/rejected": -120.316650390625, |
| "loss": 0.0058, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.8631511926651, |
| "rewards/margins": 30.350738525390625, |
| "rewards/rejected": -28.487585067749023, |
| "step": 5330 |
| }, |
| { |
| "epoch": 2.44, |
| "learning_rate": 6.24048706240487e-08, |
| "logits/chosen": -2.3006176948547363, |
| "logits/rejected": -1.861555814743042, |
| "logps/chosen": -88.1524658203125, |
| "logps/rejected": -130.68968200683594, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4345285892486572, |
| "rewards/margins": 31.54620933532715, |
| "rewards/rejected": -30.111682891845703, |
| "step": 5340 |
| }, |
| { |
| "epoch": 2.44, |
| "learning_rate": 6.189751395230847e-08, |
| "logits/chosen": -2.287721872329712, |
| "logits/rejected": -1.911665678024292, |
| "logps/chosen": -94.31678771972656, |
| "logps/rejected": -128.20156860351562, |
| "loss": 0.0021, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.2087610960006714, |
| "rewards/margins": 30.08795738220215, |
| "rewards/rejected": -28.879199981689453, |
| "step": 5350 |
| }, |
| { |
| "epoch": 2.45, |
| "learning_rate": 6.139015728056824e-08, |
| "logits/chosen": -2.2673823833465576, |
| "logits/rejected": -1.8630508184432983, |
| "logps/chosen": -85.5520248413086, |
| "logps/rejected": -127.2315444946289, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.068432331085205, |
| "rewards/margins": 31.822891235351562, |
| "rewards/rejected": -29.75446128845215, |
| "step": 5360 |
| }, |
| { |
| "epoch": 2.45, |
| "learning_rate": 6.0882800608828e-08, |
| "logits/chosen": -2.211057662963867, |
| "logits/rejected": -1.789331078529358, |
| "logps/chosen": -91.29240417480469, |
| "logps/rejected": -127.65742492675781, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5129761695861816, |
| "rewards/margins": 32.06275939941406, |
| "rewards/rejected": -29.54978370666504, |
| "step": 5370 |
| }, |
| { |
| "epoch": 2.46, |
| "learning_rate": 6.037544393708777e-08, |
| "logits/chosen": -2.1423110961914062, |
| "logits/rejected": -1.8088276386260986, |
| "logps/chosen": -86.91880798339844, |
| "logps/rejected": -131.5565948486328, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9576078653335571, |
| "rewards/margins": 31.21225929260254, |
| "rewards/rejected": -30.254650115966797, |
| "step": 5380 |
| }, |
| { |
| "epoch": 2.46, |
| "learning_rate": 5.986808726534754e-08, |
| "logits/chosen": -2.1703662872314453, |
| "logits/rejected": -1.8025754690170288, |
| "logps/chosen": -90.57283020019531, |
| "logps/rejected": -127.29478454589844, |
| "loss": 0.0057, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.42478710412979126, |
| "rewards/margins": 30.417232513427734, |
| "rewards/rejected": -29.992446899414062, |
| "step": 5390 |
| }, |
| { |
| "epoch": 2.46, |
| "learning_rate": 5.93607305936073e-08, |
| "logits/chosen": -2.2170863151550293, |
| "logits/rejected": -1.8885765075683594, |
| "logps/chosen": -83.57946014404297, |
| "logps/rejected": -127.2809829711914, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1054432392120361, |
| "rewards/margins": 31.797679901123047, |
| "rewards/rejected": -30.69223976135254, |
| "step": 5400 |
| }, |
| { |
| "epoch": 2.46, |
| "eval_logits/chosen": -2.218388795852661, |
| "eval_logits/rejected": -1.8641334772109985, |
| "eval_logps/chosen": -87.43651580810547, |
| "eval_logps/rejected": -127.63639068603516, |
| "eval_loss": 0.005585874430835247, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.6970371007919312, |
| "eval_rewards/margins": 31.099769592285156, |
| "eval_rewards/rejected": -30.402734756469727, |
| "eval_runtime": 215.095, |
| "eval_samples_per_second": 13.306, |
| "eval_steps_per_second": 0.832, |
| "step": 5400 |
| }, |
| { |
| "epoch": 2.47, |
| "learning_rate": 5.8853373921867065e-08, |
| "logits/chosen": -2.204249858856201, |
| "logits/rejected": -1.8770767450332642, |
| "logps/chosen": -89.52336120605469, |
| "logps/rejected": -134.68844604492188, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.223055362701416, |
| "rewards/margins": 32.22364044189453, |
| "rewards/rejected": -30.000585556030273, |
| "step": 5410 |
| }, |
| { |
| "epoch": 2.47, |
| "learning_rate": 5.834601725012683e-08, |
| "logits/chosen": -2.255474805831909, |
| "logits/rejected": -1.9023358821868896, |
| "logps/chosen": -89.14444732666016, |
| "logps/rejected": -128.97544860839844, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.772989273071289, |
| "rewards/margins": 31.232650756835938, |
| "rewards/rejected": -29.45966148376465, |
| "step": 5420 |
| }, |
| { |
| "epoch": 2.48, |
| "learning_rate": 5.78386605783866e-08, |
| "logits/chosen": -2.12762451171875, |
| "logits/rejected": -1.7764146327972412, |
| "logps/chosen": -89.85285186767578, |
| "logps/rejected": -135.08656311035156, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.07732389122247696, |
| "rewards/margins": 32.275455474853516, |
| "rewards/rejected": -32.19812774658203, |
| "step": 5430 |
| }, |
| { |
| "epoch": 2.48, |
| "learning_rate": 5.7331303906646365e-08, |
| "logits/chosen": -2.1940665245056152, |
| "logits/rejected": -1.8633617162704468, |
| "logps/chosen": -86.71977233886719, |
| "logps/rejected": -132.53306579589844, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.124468207359314, |
| "rewards/margins": 31.2531681060791, |
| "rewards/rejected": -30.128698348999023, |
| "step": 5440 |
| }, |
| { |
| "epoch": 2.49, |
| "learning_rate": 5.682394723490613e-08, |
| "logits/chosen": -2.319063663482666, |
| "logits/rejected": -1.9095830917358398, |
| "logps/chosen": -92.62286376953125, |
| "logps/rejected": -133.32777404785156, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9819850921630859, |
| "rewards/margins": 32.897369384765625, |
| "rewards/rejected": -31.915386199951172, |
| "step": 5450 |
| }, |
| { |
| "epoch": 2.49, |
| "learning_rate": 5.63165905631659e-08, |
| "logits/chosen": -2.249760627746582, |
| "logits/rejected": -1.880089521408081, |
| "logps/chosen": -90.4861831665039, |
| "logps/rejected": -135.6083221435547, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5621588826179504, |
| "rewards/margins": 33.28693771362305, |
| "rewards/rejected": -32.72478103637695, |
| "step": 5460 |
| }, |
| { |
| "epoch": 2.5, |
| "learning_rate": 5.5809233891425665e-08, |
| "logits/chosen": -2.229804277420044, |
| "logits/rejected": -1.8705679178237915, |
| "logps/chosen": -89.12641143798828, |
| "logps/rejected": -131.51339721679688, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6339949369430542, |
| "rewards/margins": 31.29616928100586, |
| "rewards/rejected": -30.66217613220215, |
| "step": 5470 |
| }, |
| { |
| "epoch": 2.5, |
| "learning_rate": 5.530187721968543e-08, |
| "logits/chosen": -2.2353408336639404, |
| "logits/rejected": -1.8864301443099976, |
| "logps/chosen": -90.36353302001953, |
| "logps/rejected": -135.7219696044922, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8752725720405579, |
| "rewards/margins": 33.36455535888672, |
| "rewards/rejected": -32.489280700683594, |
| "step": 5480 |
| }, |
| { |
| "epoch": 2.51, |
| "learning_rate": 5.47945205479452e-08, |
| "logits/chosen": -2.1790707111358643, |
| "logits/rejected": -1.8824752569198608, |
| "logps/chosen": -86.35264587402344, |
| "logps/rejected": -138.5924835205078, |
| "loss": 0.003, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.8381770849227905, |
| "rewards/margins": 33.54732131958008, |
| "rewards/rejected": -32.70914077758789, |
| "step": 5490 |
| }, |
| { |
| "epoch": 2.51, |
| "learning_rate": 5.4287163876204964e-08, |
| "logits/chosen": -2.253610134124756, |
| "logits/rejected": -1.8832261562347412, |
| "logps/chosen": -91.31092834472656, |
| "logps/rejected": -132.42288208007812, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8345121145248413, |
| "rewards/margins": 31.579730987548828, |
| "rewards/rejected": -30.745220184326172, |
| "step": 5500 |
| }, |
| { |
| "epoch": 2.51, |
| "eval_logits/chosen": -2.2216947078704834, |
| "eval_logits/rejected": -1.8705134391784668, |
| "eval_logps/chosen": -87.70054626464844, |
| "eval_logps/rejected": -128.5836181640625, |
| "eval_loss": 0.005452133249491453, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.56501704454422, |
| "eval_rewards/margins": 31.44136619567871, |
| "eval_rewards/rejected": -30.8763484954834, |
| "eval_runtime": 239.2992, |
| "eval_samples_per_second": 11.96, |
| "eval_steps_per_second": 0.748, |
| "step": 5500 |
| }, |
| { |
| "epoch": 2.52, |
| "learning_rate": 5.377980720446473e-08, |
| "logits/chosen": -2.198091745376587, |
| "logits/rejected": -1.8183279037475586, |
| "logps/chosen": -88.14014434814453, |
| "logps/rejected": -130.6202392578125, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0735613107681274, |
| "rewards/margins": 32.70407485961914, |
| "rewards/rejected": -31.63051414489746, |
| "step": 5510 |
| }, |
| { |
| "epoch": 2.52, |
| "learning_rate": 5.32724505327245e-08, |
| "logits/chosen": -2.260110378265381, |
| "logits/rejected": -1.8869386911392212, |
| "logps/chosen": -85.59989929199219, |
| "logps/rejected": -126.64656829833984, |
| "loss": 0.0019, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1739155054092407, |
| "rewards/margins": 29.694509506225586, |
| "rewards/rejected": -28.520593643188477, |
| "step": 5520 |
| }, |
| { |
| "epoch": 2.52, |
| "learning_rate": 5.2765093860984264e-08, |
| "logits/chosen": -2.2093660831451416, |
| "logits/rejected": -1.9314342737197876, |
| "logps/chosen": -86.80977630615234, |
| "logps/rejected": -128.8234405517578, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.0760526657104492, |
| "rewards/margins": 28.858245849609375, |
| "rewards/rejected": -29.93429946899414, |
| "step": 5530 |
| }, |
| { |
| "epoch": 2.53, |
| "learning_rate": 5.225773718924403e-08, |
| "logits/chosen": -2.182633876800537, |
| "logits/rejected": -1.857081651687622, |
| "logps/chosen": -84.45915222167969, |
| "logps/rejected": -127.63262939453125, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2383257150650024, |
| "rewards/margins": 31.82940101623535, |
| "rewards/rejected": -30.591073989868164, |
| "step": 5540 |
| }, |
| { |
| "epoch": 2.53, |
| "learning_rate": 5.17503805175038e-08, |
| "logits/chosen": -2.2090537548065186, |
| "logits/rejected": -1.8241138458251953, |
| "logps/chosen": -89.18778991699219, |
| "logps/rejected": -133.34609985351562, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9115778207778931, |
| "rewards/margins": 32.943077087402344, |
| "rewards/rejected": -32.03150177001953, |
| "step": 5550 |
| }, |
| { |
| "epoch": 2.54, |
| "learning_rate": 5.1243023845763564e-08, |
| "logits/chosen": -2.2495675086975098, |
| "logits/rejected": -1.9659570455551147, |
| "logps/chosen": -82.7123031616211, |
| "logps/rejected": -135.22415161132812, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8649141192436218, |
| "rewards/margins": 33.649383544921875, |
| "rewards/rejected": -32.78447341918945, |
| "step": 5560 |
| }, |
| { |
| "epoch": 2.54, |
| "learning_rate": 5.073566717402333e-08, |
| "logits/chosen": -2.2645201683044434, |
| "logits/rejected": -1.9173786640167236, |
| "logps/chosen": -85.26544952392578, |
| "logps/rejected": -131.0226593017578, |
| "loss": 0.0079, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9299182891845703, |
| "rewards/margins": 32.061363220214844, |
| "rewards/rejected": -31.131444931030273, |
| "step": 5570 |
| }, |
| { |
| "epoch": 2.55, |
| "learning_rate": 5.02283105022831e-08, |
| "logits/chosen": -2.21622633934021, |
| "logits/rejected": -1.8026697635650635, |
| "logps/chosen": -93.25666046142578, |
| "logps/rejected": -133.81333923339844, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4977938234806061, |
| "rewards/margins": 31.675308227539062, |
| "rewards/rejected": -31.17751693725586, |
| "step": 5580 |
| }, |
| { |
| "epoch": 2.55, |
| "learning_rate": 4.9720953830542864e-08, |
| "logits/chosen": -2.175873279571533, |
| "logits/rejected": -1.7768224477767944, |
| "logps/chosen": -91.02295684814453, |
| "logps/rejected": -131.60630798339844, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.008679199032485485, |
| "rewards/margins": 32.17238235473633, |
| "rewards/rejected": -32.181060791015625, |
| "step": 5590 |
| }, |
| { |
| "epoch": 2.56, |
| "learning_rate": 4.921359715880263e-08, |
| "logits/chosen": -2.2250924110412598, |
| "logits/rejected": -1.7964969873428345, |
| "logps/chosen": -83.45528411865234, |
| "logps/rejected": -126.2359848022461, |
| "loss": 0.0021, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5450199842453003, |
| "rewards/margins": 32.206085205078125, |
| "rewards/rejected": -30.66106605529785, |
| "step": 5600 |
| }, |
| { |
| "epoch": 2.56, |
| "eval_logits/chosen": -2.225454807281494, |
| "eval_logits/rejected": -1.8789043426513672, |
| "eval_logps/chosen": -87.96759796142578, |
| "eval_logps/rejected": -129.4971466064453, |
| "eval_loss": 0.005582863464951515, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 0.4314943850040436, |
| "eval_rewards/margins": 31.764596939086914, |
| "eval_rewards/rejected": -31.33310317993164, |
| "eval_runtime": 190.7581, |
| "eval_samples_per_second": 15.003, |
| "eval_steps_per_second": 0.938, |
| "step": 5600 |
| }, |
| { |
| "epoch": 2.56, |
| "learning_rate": 4.87062404870624e-08, |
| "logits/chosen": -2.2279891967773438, |
| "logits/rejected": -1.8970493078231812, |
| "logps/chosen": -85.4381103515625, |
| "logps/rejected": -128.7406005859375, |
| "loss": 0.002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.32868337631225586, |
| "rewards/margins": 31.653844833374023, |
| "rewards/rejected": -31.325159072875977, |
| "step": 5610 |
| }, |
| { |
| "epoch": 2.57, |
| "learning_rate": 4.8198883815322164e-08, |
| "logits/chosen": -2.2391271591186523, |
| "logits/rejected": -1.919921636581421, |
| "logps/chosen": -86.07292175292969, |
| "logps/rejected": -133.414306640625, |
| "loss": 0.0064, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.4069444537162781, |
| "rewards/margins": 31.80039405822754, |
| "rewards/rejected": -31.393451690673828, |
| "step": 5620 |
| }, |
| { |
| "epoch": 2.57, |
| "learning_rate": 4.769152714358193e-08, |
| "logits/chosen": -2.1675751209259033, |
| "logits/rejected": -1.8634262084960938, |
| "logps/chosen": -84.79447937011719, |
| "logps/rejected": -135.7115020751953, |
| "loss": 0.0009, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.24717219173908234, |
| "rewards/margins": 33.35146713256836, |
| "rewards/rejected": -33.10429382324219, |
| "step": 5630 |
| }, |
| { |
| "epoch": 2.57, |
| "learning_rate": 4.71841704718417e-08, |
| "logits/chosen": -2.1868836879730225, |
| "logits/rejected": -1.8967559337615967, |
| "logps/chosen": -88.42156982421875, |
| "logps/rejected": -132.83580017089844, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.0003570079861674458, |
| "rewards/margins": 31.02736473083496, |
| "rewards/rejected": -31.027725219726562, |
| "step": 5640 |
| }, |
| { |
| "epoch": 2.58, |
| "learning_rate": 4.6676813800101464e-08, |
| "logits/chosen": -2.281507968902588, |
| "logits/rejected": -1.8936166763305664, |
| "logps/chosen": -91.97183990478516, |
| "logps/rejected": -128.8743438720703, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.5036630630493164, |
| "rewards/margins": 31.219980239868164, |
| "rewards/rejected": -29.716318130493164, |
| "step": 5650 |
| }, |
| { |
| "epoch": 2.58, |
| "learning_rate": 4.616945712836123e-08, |
| "logits/chosen": -2.1977477073669434, |
| "logits/rejected": -1.8735698461532593, |
| "logps/chosen": -89.23414611816406, |
| "logps/rejected": -131.9024658203125, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5118563771247864, |
| "rewards/margins": 32.36008834838867, |
| "rewards/rejected": -31.848236083984375, |
| "step": 5660 |
| }, |
| { |
| "epoch": 2.59, |
| "learning_rate": 4.5662100456621e-08, |
| "logits/chosen": -2.2489736080169678, |
| "logits/rejected": -1.8674437999725342, |
| "logps/chosen": -87.26219940185547, |
| "logps/rejected": -134.82095336914062, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8911517858505249, |
| "rewards/margins": 32.76057434082031, |
| "rewards/rejected": -31.86942481994629, |
| "step": 5670 |
| }, |
| { |
| "epoch": 2.59, |
| "learning_rate": 4.5154743784880764e-08, |
| "logits/chosen": -2.1873042583465576, |
| "logits/rejected": -1.924572229385376, |
| "logps/chosen": -79.94978332519531, |
| "logps/rejected": -128.44432067871094, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.669002890586853, |
| "rewards/margins": 30.952983856201172, |
| "rewards/rejected": -30.283981323242188, |
| "step": 5680 |
| }, |
| { |
| "epoch": 2.6, |
| "learning_rate": 4.464738711314053e-08, |
| "logits/chosen": -2.160274028778076, |
| "logits/rejected": -1.8004605770111084, |
| "logps/chosen": -96.46055603027344, |
| "logps/rejected": -133.01870727539062, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.372810959815979, |
| "rewards/margins": 31.60434913635254, |
| "rewards/rejected": -30.231542587280273, |
| "step": 5690 |
| }, |
| { |
| "epoch": 2.6, |
| "learning_rate": 4.41400304414003e-08, |
| "logits/chosen": -2.2056326866149902, |
| "logits/rejected": -1.73297917842865, |
| "logps/chosen": -96.11383056640625, |
| "logps/rejected": -131.00546264648438, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5699256658554077, |
| "rewards/margins": 32.902523040771484, |
| "rewards/rejected": -31.332595825195312, |
| "step": 5700 |
| }, |
| { |
| "epoch": 2.6, |
| "eval_logits/chosen": -2.226090669631958, |
| "eval_logits/rejected": -1.876379370689392, |
| "eval_logps/chosen": -87.41751861572266, |
| "eval_logps/rejected": -128.55892944335938, |
| "eval_loss": 0.005494570359587669, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 0.7065298557281494, |
| "eval_rewards/margins": 31.570541381835938, |
| "eval_rewards/rejected": -30.864015579223633, |
| "eval_runtime": 195.492, |
| "eval_samples_per_second": 14.64, |
| "eval_steps_per_second": 0.916, |
| "step": 5700 |
| }, |
| { |
| "epoch": 2.61, |
| "learning_rate": 4.3632673769660064e-08, |
| "logits/chosen": -2.2404072284698486, |
| "logits/rejected": -1.8427753448486328, |
| "logps/chosen": -88.0647964477539, |
| "logps/rejected": -132.00401306152344, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0430487394332886, |
| "rewards/margins": 31.766735076904297, |
| "rewards/rejected": -30.723682403564453, |
| "step": 5710 |
| }, |
| { |
| "epoch": 2.61, |
| "learning_rate": 4.312531709791983e-08, |
| "logits/chosen": -2.238086223602295, |
| "logits/rejected": -1.9353258609771729, |
| "logps/chosen": -86.8983383178711, |
| "logps/rejected": -134.00552368164062, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7805923223495483, |
| "rewards/margins": 32.820377349853516, |
| "rewards/rejected": -32.03978729248047, |
| "step": 5720 |
| }, |
| { |
| "epoch": 2.62, |
| "learning_rate": 4.26179604261796e-08, |
| "logits/chosen": -2.210732936859131, |
| "logits/rejected": -1.8510334491729736, |
| "logps/chosen": -86.54997253417969, |
| "logps/rejected": -128.822998046875, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1234691143035889, |
| "rewards/margins": 31.587039947509766, |
| "rewards/rejected": -30.46356773376465, |
| "step": 5730 |
| }, |
| { |
| "epoch": 2.62, |
| "learning_rate": 4.2110603754439363e-08, |
| "logits/chosen": -2.271686553955078, |
| "logits/rejected": -1.9221289157867432, |
| "logps/chosen": -85.12126922607422, |
| "logps/rejected": -132.54244995117188, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.38059353828430176, |
| "rewards/margins": 32.59527587890625, |
| "rewards/rejected": -32.214683532714844, |
| "step": 5740 |
| }, |
| { |
| "epoch": 2.62, |
| "learning_rate": 4.160324708269913e-08, |
| "logits/chosen": -2.2383780479431152, |
| "logits/rejected": -1.9224342107772827, |
| "logps/chosen": -87.90855407714844, |
| "logps/rejected": -133.9766387939453, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.365673303604126, |
| "rewards/margins": 32.20100784301758, |
| "rewards/rejected": -30.8353328704834, |
| "step": 5750 |
| }, |
| { |
| "epoch": 2.63, |
| "learning_rate": 4.10958904109589e-08, |
| "logits/chosen": -2.2130231857299805, |
| "logits/rejected": -1.9353950023651123, |
| "logps/chosen": -79.39093017578125, |
| "logps/rejected": -129.04461669921875, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7016046047210693, |
| "rewards/margins": 31.313640594482422, |
| "rewards/rejected": -29.612030029296875, |
| "step": 5760 |
| }, |
| { |
| "epoch": 2.63, |
| "learning_rate": 4.0588533739218663e-08, |
| "logits/chosen": -2.196366786956787, |
| "logits/rejected": -1.783656358718872, |
| "logps/chosen": -81.48753356933594, |
| "logps/rejected": -127.0324478149414, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7704079151153564, |
| "rewards/margins": 33.856910705566406, |
| "rewards/rejected": -32.08650207519531, |
| "step": 5770 |
| }, |
| { |
| "epoch": 2.64, |
| "learning_rate": 4.0081177067478437e-08, |
| "logits/chosen": -2.1401500701904297, |
| "logits/rejected": -1.800840139389038, |
| "logps/chosen": -90.09111785888672, |
| "logps/rejected": -134.8428955078125, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.179386854171753, |
| "rewards/margins": 33.37128448486328, |
| "rewards/rejected": -32.191898345947266, |
| "step": 5780 |
| }, |
| { |
| "epoch": 2.64, |
| "learning_rate": 3.95738203957382e-08, |
| "logits/chosen": -2.2394440174102783, |
| "logits/rejected": -1.904920220375061, |
| "logps/chosen": -87.17567443847656, |
| "logps/rejected": -130.36404418945312, |
| "loss": 0.0054, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.23616953194141388, |
| "rewards/margins": 31.9814453125, |
| "rewards/rejected": -31.74527931213379, |
| "step": 5790 |
| }, |
| { |
| "epoch": 2.65, |
| "learning_rate": 3.906646372399797e-08, |
| "logits/chosen": -2.1932971477508545, |
| "logits/rejected": -1.808300256729126, |
| "logps/chosen": -90.1788558959961, |
| "logps/rejected": -135.00550842285156, |
| "loss": 0.0055, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.4655330777168274, |
| "rewards/margins": 31.84645652770996, |
| "rewards/rejected": -31.380924224853516, |
| "step": 5800 |
| }, |
| { |
| "epoch": 2.65, |
| "eval_logits/chosen": -2.2272167205810547, |
| "eval_logits/rejected": -1.878515601158142, |
| "eval_logps/chosen": -87.6694564819336, |
| "eval_logps/rejected": -129.5568084716797, |
| "eval_loss": 0.005615293048322201, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.5805642604827881, |
| "eval_rewards/margins": 31.943510055541992, |
| "eval_rewards/rejected": -31.36294937133789, |
| "eval_runtime": 188.904, |
| "eval_samples_per_second": 15.151, |
| "eval_steps_per_second": 0.948, |
| "step": 5800 |
| }, |
| { |
| "epoch": 2.65, |
| "learning_rate": 3.8559107052257736e-08, |
| "logits/chosen": -2.2895407676696777, |
| "logits/rejected": -1.9646167755126953, |
| "logps/chosen": -84.39744567871094, |
| "logps/rejected": -128.11642456054688, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.45202645659446716, |
| "rewards/margins": 31.760883331298828, |
| "rewards/rejected": -32.21290969848633, |
| "step": 5810 |
| }, |
| { |
| "epoch": 2.66, |
| "learning_rate": 3.80517503805175e-08, |
| "logits/chosen": -2.2432637214660645, |
| "logits/rejected": -1.90109384059906, |
| "logps/chosen": -87.83378601074219, |
| "logps/rejected": -134.02479553222656, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.3190554678440094, |
| "rewards/margins": 32.41364669799805, |
| "rewards/rejected": -32.73270034790039, |
| "step": 5820 |
| }, |
| { |
| "epoch": 2.66, |
| "learning_rate": 3.754439370877727e-08, |
| "logits/chosen": -2.249756336212158, |
| "logits/rejected": -1.8411035537719727, |
| "logps/chosen": -92.8016128540039, |
| "logps/rejected": -141.8028106689453, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7895019054412842, |
| "rewards/margins": 35.841468811035156, |
| "rewards/rejected": -34.051971435546875, |
| "step": 5830 |
| }, |
| { |
| "epoch": 2.67, |
| "learning_rate": 3.7037037037037036e-08, |
| "logits/chosen": -2.2434568405151367, |
| "logits/rejected": -1.8818607330322266, |
| "logps/chosen": -85.84037780761719, |
| "logps/rejected": -130.4074249267578, |
| "loss": 0.0013, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.18663930892944336, |
| "rewards/margins": 32.276615142822266, |
| "rewards/rejected": -32.08997344970703, |
| "step": 5840 |
| }, |
| { |
| "epoch": 2.67, |
| "learning_rate": 3.65296803652968e-08, |
| "logits/chosen": -2.266979932785034, |
| "logits/rejected": -1.9421230554580688, |
| "logps/chosen": -88.89350128173828, |
| "logps/rejected": -130.6448516845703, |
| "loss": 0.0059, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.22369590401649475, |
| "rewards/margins": 30.364971160888672, |
| "rewards/rejected": -30.14127540588379, |
| "step": 5850 |
| }, |
| { |
| "epoch": 2.67, |
| "learning_rate": 3.602232369355657e-08, |
| "logits/chosen": -2.245170831680298, |
| "logits/rejected": -1.8216416835784912, |
| "logps/chosen": -89.65043640136719, |
| "logps/rejected": -136.75848388671875, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8733927607536316, |
| "rewards/margins": 34.61988830566406, |
| "rewards/rejected": -33.746498107910156, |
| "step": 5860 |
| }, |
| { |
| "epoch": 2.68, |
| "learning_rate": 3.5514967021816336e-08, |
| "logits/chosen": -2.264880895614624, |
| "logits/rejected": -1.87582528591156, |
| "logps/chosen": -89.56029510498047, |
| "logps/rejected": -132.30987548828125, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9222901463508606, |
| "rewards/margins": 32.37554931640625, |
| "rewards/rejected": -31.453258514404297, |
| "step": 5870 |
| }, |
| { |
| "epoch": 2.68, |
| "learning_rate": 3.50076103500761e-08, |
| "logits/chosen": -2.2106261253356934, |
| "logits/rejected": -1.8250715732574463, |
| "logps/chosen": -86.20150756835938, |
| "logps/rejected": -131.51016235351562, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.9935638904571533, |
| "rewards/margins": 32.885887145996094, |
| "rewards/rejected": -30.892318725585938, |
| "step": 5880 |
| }, |
| { |
| "epoch": 2.69, |
| "learning_rate": 3.450025367833587e-08, |
| "logits/chosen": -2.1880767345428467, |
| "logits/rejected": -1.850630521774292, |
| "logps/chosen": -92.76158142089844, |
| "logps/rejected": -135.90554809570312, |
| "loss": 0.008, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7444310188293457, |
| "rewards/margins": 32.634525299072266, |
| "rewards/rejected": -31.890094757080078, |
| "step": 5890 |
| }, |
| { |
| "epoch": 2.69, |
| "learning_rate": 3.3992897006595636e-08, |
| "logits/chosen": -2.2573089599609375, |
| "logits/rejected": -1.9743645191192627, |
| "logps/chosen": -89.33732604980469, |
| "logps/rejected": -133.85531616210938, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.07906317710876465, |
| "rewards/margins": 31.270915985107422, |
| "rewards/rejected": -31.191858291625977, |
| "step": 5900 |
| }, |
| { |
| "epoch": 2.69, |
| "eval_logits/chosen": -2.2270514965057373, |
| "eval_logits/rejected": -1.8773261308670044, |
| "eval_logps/chosen": -88.0054931640625, |
| "eval_logps/rejected": -130.35690307617188, |
| "eval_loss": 0.0056848106905817986, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.4125469923019409, |
| "eval_rewards/margins": 32.175540924072266, |
| "eval_rewards/rejected": -31.76299285888672, |
| "eval_runtime": 187.4914, |
| "eval_samples_per_second": 15.265, |
| "eval_steps_per_second": 0.955, |
| "step": 5900 |
| }, |
| { |
| "epoch": 2.7, |
| "learning_rate": 3.34855403348554e-08, |
| "logits/chosen": -2.249181032180786, |
| "logits/rejected": -1.8485383987426758, |
| "logps/chosen": -94.90681457519531, |
| "logps/rejected": -131.73171997070312, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9341878890991211, |
| "rewards/margins": 31.412506103515625, |
| "rewards/rejected": -30.478321075439453, |
| "step": 5910 |
| }, |
| { |
| "epoch": 2.7, |
| "learning_rate": 3.297818366311517e-08, |
| "logits/chosen": -2.2404422760009766, |
| "logits/rejected": -1.9138734340667725, |
| "logps/chosen": -84.72028350830078, |
| "logps/rejected": -135.90516662597656, |
| "loss": 0.0042, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.11444835364818573, |
| "rewards/margins": 33.07588577270508, |
| "rewards/rejected": -32.96143341064453, |
| "step": 5920 |
| }, |
| { |
| "epoch": 2.71, |
| "learning_rate": 3.2470826991374936e-08, |
| "logits/chosen": -2.2721214294433594, |
| "logits/rejected": -1.8678615093231201, |
| "logps/chosen": -89.56704711914062, |
| "logps/rejected": -135.8103790283203, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.21784238517284393, |
| "rewards/margins": 33.14167022705078, |
| "rewards/rejected": -32.923828125, |
| "step": 5930 |
| }, |
| { |
| "epoch": 2.71, |
| "learning_rate": 3.19634703196347e-08, |
| "logits/chosen": -2.2767839431762695, |
| "logits/rejected": -1.95871102809906, |
| "logps/chosen": -92.42479705810547, |
| "logps/rejected": -139.7154998779297, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1621972620487213, |
| "rewards/margins": 33.40778350830078, |
| "rewards/rejected": -33.56998062133789, |
| "step": 5940 |
| }, |
| { |
| "epoch": 2.72, |
| "learning_rate": 3.145611364789447e-08, |
| "logits/chosen": -2.1546614170074463, |
| "logits/rejected": -1.7696462869644165, |
| "logps/chosen": -92.0521469116211, |
| "logps/rejected": -134.0331268310547, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3609464168548584, |
| "rewards/margins": 32.90177917480469, |
| "rewards/rejected": -31.540836334228516, |
| "step": 5950 |
| }, |
| { |
| "epoch": 2.72, |
| "learning_rate": 3.0948756976154236e-08, |
| "logits/chosen": -2.2606728076934814, |
| "logits/rejected": -1.9319846630096436, |
| "logps/chosen": -84.58219146728516, |
| "logps/rejected": -129.25262451171875, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3570373058319092, |
| "rewards/margins": 32.03596878051758, |
| "rewards/rejected": -30.678930282592773, |
| "step": 5960 |
| }, |
| { |
| "epoch": 2.73, |
| "learning_rate": 3.0441400304414e-08, |
| "logits/chosen": -2.2868599891662598, |
| "logits/rejected": -1.9675430059432983, |
| "logps/chosen": -84.88082122802734, |
| "logps/rejected": -130.11279296875, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.3799717426300049, |
| "rewards/margins": 32.025333404541016, |
| "rewards/rejected": -30.645360946655273, |
| "step": 5970 |
| }, |
| { |
| "epoch": 2.73, |
| "learning_rate": 2.993404363267377e-08, |
| "logits/chosen": -2.2390341758728027, |
| "logits/rejected": -1.9002273082733154, |
| "logps/chosen": -84.3361587524414, |
| "logps/rejected": -132.21054077148438, |
| "loss": 0.0015, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.16436767578125, |
| "rewards/margins": 31.722976684570312, |
| "rewards/rejected": -31.887344360351562, |
| "step": 5980 |
| }, |
| { |
| "epoch": 2.73, |
| "learning_rate": 2.9426686960933532e-08, |
| "logits/chosen": -2.199117660522461, |
| "logits/rejected": -1.7541431188583374, |
| "logps/chosen": -94.94574737548828, |
| "logps/rejected": -130.81954956054688, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6142606735229492, |
| "rewards/margins": 33.10905456542969, |
| "rewards/rejected": -31.494792938232422, |
| "step": 5990 |
| }, |
| { |
| "epoch": 2.74, |
| "learning_rate": 2.89193302891933e-08, |
| "logits/chosen": -2.368619918823242, |
| "logits/rejected": -1.9851986169815063, |
| "logps/chosen": -90.95460510253906, |
| "logps/rejected": -135.75814819335938, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2684900164604187, |
| "rewards/margins": 33.65724563598633, |
| "rewards/rejected": -33.388755798339844, |
| "step": 6000 |
| }, |
| { |
| "epoch": 2.74, |
| "eval_logits/chosen": -2.220628023147583, |
| "eval_logits/rejected": -1.8684388399124146, |
| "eval_logps/chosen": -87.90426635742188, |
| "eval_logps/rejected": -129.78916931152344, |
| "eval_loss": 0.005480717867612839, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.4631572663784027, |
| "eval_rewards/margins": 31.94228172302246, |
| "eval_rewards/rejected": -31.479124069213867, |
| "eval_runtime": 188.1705, |
| "eval_samples_per_second": 15.21, |
| "eval_steps_per_second": 0.951, |
| "step": 6000 |
| }, |
| { |
| "epoch": 2.74, |
| "learning_rate": 2.8411973617453066e-08, |
| "logits/chosen": -2.1487109661102295, |
| "logits/rejected": -1.817098617553711, |
| "logps/chosen": -88.42924499511719, |
| "logps/rejected": -132.3211212158203, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3025569915771484, |
| "rewards/margins": 33.558433532714844, |
| "rewards/rejected": -32.25587844848633, |
| "step": 6010 |
| }, |
| { |
| "epoch": 2.75, |
| "learning_rate": 2.7904616945712832e-08, |
| "logits/chosen": -2.2678167819976807, |
| "logits/rejected": -1.8667113780975342, |
| "logps/chosen": -86.5342025756836, |
| "logps/rejected": -129.55935668945312, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0008518695831299, |
| "rewards/margins": 32.452972412109375, |
| "rewards/rejected": -31.452117919921875, |
| "step": 6020 |
| }, |
| { |
| "epoch": 2.75, |
| "learning_rate": 2.73972602739726e-08, |
| "logits/chosen": -2.264303684234619, |
| "logits/rejected": -1.878883957862854, |
| "logps/chosen": -87.34925842285156, |
| "logps/rejected": -130.01307678222656, |
| "loss": 0.0031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4018937945365906, |
| "rewards/margins": 32.04921340942383, |
| "rewards/rejected": -31.647314071655273, |
| "step": 6030 |
| }, |
| { |
| "epoch": 2.76, |
| "learning_rate": 2.6889903602232366e-08, |
| "logits/chosen": -2.2151219844818115, |
| "logits/rejected": -1.8500292301177979, |
| "logps/chosen": -89.37690734863281, |
| "logps/rejected": -132.4504852294922, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7266126871109009, |
| "rewards/margins": 33.85057067871094, |
| "rewards/rejected": -32.123958587646484, |
| "step": 6040 |
| }, |
| { |
| "epoch": 2.76, |
| "learning_rate": 2.6382546930492132e-08, |
| "logits/chosen": -2.3891372680664062, |
| "logits/rejected": -2.01668119430542, |
| "logps/chosen": -88.968017578125, |
| "logps/rejected": -134.05789184570312, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.28615638613700867, |
| "rewards/margins": 31.448923110961914, |
| "rewards/rejected": -31.162769317626953, |
| "step": 6050 |
| }, |
| { |
| "epoch": 2.77, |
| "learning_rate": 2.58751902587519e-08, |
| "logits/chosen": -2.265878200531006, |
| "logits/rejected": -1.890244722366333, |
| "logps/chosen": -83.14924621582031, |
| "logps/rejected": -128.8566436767578, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.234644889831543, |
| "rewards/margins": 32.04364776611328, |
| "rewards/rejected": -30.80900001525879, |
| "step": 6060 |
| }, |
| { |
| "epoch": 2.77, |
| "learning_rate": 2.5367833587011665e-08, |
| "logits/chosen": -2.3138954639434814, |
| "logits/rejected": -1.8876469135284424, |
| "logps/chosen": -96.02142333984375, |
| "logps/rejected": -132.93771362304688, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.01418936252594, |
| "rewards/margins": 31.96274185180664, |
| "rewards/rejected": -30.948551177978516, |
| "step": 6070 |
| }, |
| { |
| "epoch": 2.78, |
| "learning_rate": 2.4860476915271432e-08, |
| "logits/chosen": -2.2674362659454346, |
| "logits/rejected": -1.8280704021453857, |
| "logps/chosen": -91.15046691894531, |
| "logps/rejected": -131.0101318359375, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.064391613006592, |
| "rewards/margins": 34.05684280395508, |
| "rewards/rejected": -31.99245262145996, |
| "step": 6080 |
| }, |
| { |
| "epoch": 2.78, |
| "learning_rate": 2.43531202435312e-08, |
| "logits/chosen": -2.2650763988494873, |
| "logits/rejected": -1.9617116451263428, |
| "logps/chosen": -81.56208801269531, |
| "logps/rejected": -131.12905883789062, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9505952000617981, |
| "rewards/margins": 32.1887092590332, |
| "rewards/rejected": -31.238117218017578, |
| "step": 6090 |
| }, |
| { |
| "epoch": 2.78, |
| "learning_rate": 2.3845763571790965e-08, |
| "logits/chosen": -2.2631096839904785, |
| "logits/rejected": -1.8503172397613525, |
| "logps/chosen": -93.38179016113281, |
| "logps/rejected": -127.70606994628906, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.097931981086731, |
| "rewards/margins": 31.50238037109375, |
| "rewards/rejected": -30.40444564819336, |
| "step": 6100 |
| }, |
| { |
| "epoch": 2.78, |
| "eval_logits/chosen": -2.2261240482330322, |
| "eval_logits/rejected": -1.877672791481018, |
| "eval_logps/chosen": -87.89844512939453, |
| "eval_logps/rejected": -130.31732177734375, |
| "eval_loss": 0.005521238315850496, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.46606892347335815, |
| "eval_rewards/margins": 32.209266662597656, |
| "eval_rewards/rejected": -31.743196487426758, |
| "eval_runtime": 230.5005, |
| "eval_samples_per_second": 12.416, |
| "eval_steps_per_second": 0.777, |
| "step": 6100 |
| }, |
| { |
| "epoch": 2.79, |
| "learning_rate": 2.3338406900050732e-08, |
| "logits/chosen": -2.156193494796753, |
| "logits/rejected": -1.7497676610946655, |
| "logps/chosen": -88.01410675048828, |
| "logps/rejected": -129.71466064453125, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.868211567401886, |
| "rewards/margins": 32.48591995239258, |
| "rewards/rejected": -31.61771011352539, |
| "step": 6110 |
| }, |
| { |
| "epoch": 2.79, |
| "learning_rate": 2.28310502283105e-08, |
| "logits/chosen": -2.27543306350708, |
| "logits/rejected": -1.8602546453475952, |
| "logps/chosen": -87.84466552734375, |
| "logps/rejected": -132.81777954101562, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.908778429031372, |
| "rewards/margins": 33.64677810668945, |
| "rewards/rejected": -31.73800277709961, |
| "step": 6120 |
| }, |
| { |
| "epoch": 2.8, |
| "learning_rate": 2.2323693556570265e-08, |
| "logits/chosen": -2.2359254360198975, |
| "logits/rejected": -1.9188182353973389, |
| "logps/chosen": -83.94422149658203, |
| "logps/rejected": -129.99581909179688, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0997272729873657, |
| "rewards/margins": 33.11286163330078, |
| "rewards/rejected": -32.01313400268555, |
| "step": 6130 |
| }, |
| { |
| "epoch": 2.8, |
| "learning_rate": 2.1816336884830032e-08, |
| "logits/chosen": -2.220909357070923, |
| "logits/rejected": -1.9032386541366577, |
| "logps/chosen": -84.5809555053711, |
| "logps/rejected": -131.4840545654297, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.0809751749038696, |
| "rewards/margins": 32.72708511352539, |
| "rewards/rejected": -31.6461124420166, |
| "step": 6140 |
| }, |
| { |
| "epoch": 2.81, |
| "learning_rate": 2.13089802130898e-08, |
| "logits/chosen": -2.242598056793213, |
| "logits/rejected": -1.8045127391815186, |
| "logps/chosen": -88.67266082763672, |
| "logps/rejected": -134.9513397216797, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.345744013786316, |
| "rewards/margins": 35.6363410949707, |
| "rewards/rejected": -34.29059600830078, |
| "step": 6150 |
| }, |
| { |
| "epoch": 2.81, |
| "learning_rate": 2.0801623541349565e-08, |
| "logits/chosen": -2.2422022819519043, |
| "logits/rejected": -1.8511192798614502, |
| "logps/chosen": -89.98072052001953, |
| "logps/rejected": -134.96629333496094, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8219916820526123, |
| "rewards/margins": 34.537723541259766, |
| "rewards/rejected": -32.71573257446289, |
| "step": 6160 |
| }, |
| { |
| "epoch": 2.82, |
| "learning_rate": 2.0294266869609332e-08, |
| "logits/chosen": -2.2780232429504395, |
| "logits/rejected": -1.8996295928955078, |
| "logps/chosen": -90.99410247802734, |
| "logps/rejected": -132.14706420898438, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.23341834545135498, |
| "rewards/margins": 32.95896530151367, |
| "rewards/rejected": -32.725547790527344, |
| "step": 6170 |
| }, |
| { |
| "epoch": 2.82, |
| "learning_rate": 1.97869101978691e-08, |
| "logits/chosen": -2.265056610107422, |
| "logits/rejected": -1.8394749164581299, |
| "logps/chosen": -93.9018783569336, |
| "logps/rejected": -132.54859924316406, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5913444757461548, |
| "rewards/margins": 33.28534698486328, |
| "rewards/rejected": -32.694007873535156, |
| "step": 6180 |
| }, |
| { |
| "epoch": 2.83, |
| "learning_rate": 1.9279553526128868e-08, |
| "logits/chosen": -2.234004259109497, |
| "logits/rejected": -1.9320383071899414, |
| "logps/chosen": -80.52074432373047, |
| "logps/rejected": -131.88790893554688, |
| "loss": 0.0022, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.3767390549182892, |
| "rewards/margins": 33.22993087768555, |
| "rewards/rejected": -32.85319137573242, |
| "step": 6190 |
| }, |
| { |
| "epoch": 2.83, |
| "learning_rate": 1.8772196854388635e-08, |
| "logits/chosen": -2.2175235748291016, |
| "logits/rejected": -1.8579527139663696, |
| "logps/chosen": -86.18971252441406, |
| "logps/rejected": -130.14015197753906, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.16683992743492126, |
| "rewards/margins": 31.512447357177734, |
| "rewards/rejected": -31.67928695678711, |
| "step": 6200 |
| }, |
| { |
| "epoch": 2.83, |
| "eval_logits/chosen": -2.2242860794067383, |
| "eval_logits/rejected": -1.8746119737625122, |
| "eval_logps/chosen": -87.8666763305664, |
| "eval_logps/rejected": -130.38250732421875, |
| "eval_loss": 0.005393806379288435, |
| "eval_rewards/accuracies": 0.9972066879272461, |
| "eval_rewards/chosen": 0.48195433616638184, |
| "eval_rewards/margins": 32.25774383544922, |
| "eval_rewards/rejected": -31.775789260864258, |
| "eval_runtime": 225.1441, |
| "eval_samples_per_second": 12.712, |
| "eval_steps_per_second": 0.795, |
| "step": 6200 |
| }, |
| { |
| "epoch": 2.83, |
| "learning_rate": 1.82648401826484e-08, |
| "logits/chosen": -2.2525382041931152, |
| "logits/rejected": -1.9172885417938232, |
| "logps/chosen": -85.48805236816406, |
| "logps/rejected": -133.95077514648438, |
| "loss": 0.0054, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1090790256857872, |
| "rewards/margins": 32.306297302246094, |
| "rewards/rejected": -32.19722366333008, |
| "step": 6210 |
| }, |
| { |
| "epoch": 2.84, |
| "learning_rate": 1.7757483510908168e-08, |
| "logits/chosen": -2.1963915824890137, |
| "logits/rejected": -1.8396198749542236, |
| "logps/chosen": -87.08211517333984, |
| "logps/rejected": -136.7283477783203, |
| "loss": 0.0023, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.9704242944717407, |
| "rewards/margins": 33.93294906616211, |
| "rewards/rejected": -32.96253204345703, |
| "step": 6220 |
| }, |
| { |
| "epoch": 2.84, |
| "learning_rate": 1.7250126839167935e-08, |
| "logits/chosen": -2.220945358276367, |
| "logits/rejected": -1.8874809741973877, |
| "logps/chosen": -83.79931640625, |
| "logps/rejected": -129.1210174560547, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5613209009170532, |
| "rewards/margins": 32.765995025634766, |
| "rewards/rejected": -32.204673767089844, |
| "step": 6230 |
| }, |
| { |
| "epoch": 2.85, |
| "learning_rate": 1.67427701674277e-08, |
| "logits/chosen": -2.2060558795928955, |
| "logits/rejected": -1.8112850189208984, |
| "logps/chosen": -87.95478820800781, |
| "logps/rejected": -132.51715087890625, |
| "loss": 0.0023, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4738547801971436, |
| "rewards/margins": 34.16747283935547, |
| "rewards/rejected": -32.69361877441406, |
| "step": 6240 |
| }, |
| { |
| "epoch": 2.85, |
| "learning_rate": 1.6235413495687468e-08, |
| "logits/chosen": -2.1749913692474365, |
| "logits/rejected": -1.8666985034942627, |
| "logps/chosen": -88.18854522705078, |
| "logps/rejected": -133.01121520996094, |
| "loss": 0.0024, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7993472218513489, |
| "rewards/margins": 31.564916610717773, |
| "rewards/rejected": -30.765567779541016, |
| "step": 6250 |
| }, |
| { |
| "epoch": 2.86, |
| "learning_rate": 1.5728056823947235e-08, |
| "logits/chosen": -2.161698341369629, |
| "logits/rejected": -1.7702264785766602, |
| "logps/chosen": -83.78431701660156, |
| "logps/rejected": -125.90284729003906, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6891142129898071, |
| "rewards/margins": 30.822668075561523, |
| "rewards/rejected": -30.133554458618164, |
| "step": 6260 |
| }, |
| { |
| "epoch": 2.86, |
| "learning_rate": 1.5220700152207e-08, |
| "logits/chosen": -2.3119311332702637, |
| "logits/rejected": -1.9453165531158447, |
| "logps/chosen": -87.14160919189453, |
| "logps/rejected": -134.0089569091797, |
| "loss": 0.0014, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7904703617095947, |
| "rewards/margins": 33.403202056884766, |
| "rewards/rejected": -32.61273193359375, |
| "step": 6270 |
| }, |
| { |
| "epoch": 2.87, |
| "learning_rate": 1.4713343480466766e-08, |
| "logits/chosen": -2.2479605674743652, |
| "logits/rejected": -1.8846473693847656, |
| "logps/chosen": -94.47314453125, |
| "logps/rejected": -134.31393432617188, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.504614531993866, |
| "rewards/margins": 31.21249008178711, |
| "rewards/rejected": -31.71710205078125, |
| "step": 6280 |
| }, |
| { |
| "epoch": 2.87, |
| "learning_rate": 1.4205986808726533e-08, |
| "logits/chosen": -2.1867895126342773, |
| "logits/rejected": -1.8338983058929443, |
| "logps/chosen": -86.80367279052734, |
| "logps/rejected": -134.02243041992188, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9545267820358276, |
| "rewards/margins": 32.74871826171875, |
| "rewards/rejected": -31.794189453125, |
| "step": 6290 |
| }, |
| { |
| "epoch": 2.88, |
| "learning_rate": 1.36986301369863e-08, |
| "logits/chosen": -2.277268648147583, |
| "logits/rejected": -1.94185471534729, |
| "logps/chosen": -86.12911224365234, |
| "logps/rejected": -135.1995849609375, |
| "loss": 0.0017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7337052822113037, |
| "rewards/margins": 33.44802474975586, |
| "rewards/rejected": -32.71432113647461, |
| "step": 6300 |
| }, |
| { |
| "epoch": 2.88, |
| "eval_logits/chosen": -2.2262394428253174, |
| "eval_logits/rejected": -1.8772982358932495, |
| "eval_logps/chosen": -87.41510772705078, |
| "eval_logps/rejected": -129.4889373779297, |
| "eval_loss": 0.005471652373671532, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.7077398300170898, |
| "eval_rewards/margins": 32.0367431640625, |
| "eval_rewards/rejected": -31.329004287719727, |
| "eval_runtime": 218.9647, |
| "eval_samples_per_second": 13.071, |
| "eval_steps_per_second": 0.817, |
| "step": 6300 |
| }, |
| { |
| "epoch": 2.88, |
| "learning_rate": 1.3191273465246066e-08, |
| "logits/chosen": -2.2694547176361084, |
| "logits/rejected": -1.8749288320541382, |
| "logps/chosen": -86.23151397705078, |
| "logps/rejected": -126.11222076416016, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5606023073196411, |
| "rewards/margins": 32.055213928222656, |
| "rewards/rejected": -30.49460792541504, |
| "step": 6310 |
| }, |
| { |
| "epoch": 2.88, |
| "learning_rate": 1.2683916793505833e-08, |
| "logits/chosen": -2.271695375442505, |
| "logits/rejected": -1.8720725774765015, |
| "logps/chosen": -88.22488403320312, |
| "logps/rejected": -133.49237060546875, |
| "loss": 0.0033, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0623927116394043, |
| "rewards/margins": 33.233009338378906, |
| "rewards/rejected": -32.170616149902344, |
| "step": 6320 |
| }, |
| { |
| "epoch": 2.89, |
| "learning_rate": 1.21765601217656e-08, |
| "logits/chosen": -2.2026755809783936, |
| "logits/rejected": -1.8335577249526978, |
| "logps/chosen": -86.81916809082031, |
| "logps/rejected": -130.44638061523438, |
| "loss": 0.0064, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3268142938613892, |
| "rewards/margins": 32.48900604248047, |
| "rewards/rejected": -31.16219139099121, |
| "step": 6330 |
| }, |
| { |
| "epoch": 2.89, |
| "learning_rate": 1.1669203450025366e-08, |
| "logits/chosen": -2.2959322929382324, |
| "logits/rejected": -1.9061601161956787, |
| "logps/chosen": -92.18937683105469, |
| "logps/rejected": -137.560302734375, |
| "loss": 0.0043, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8249843120574951, |
| "rewards/margins": 33.91102600097656, |
| "rewards/rejected": -32.08604049682617, |
| "step": 6340 |
| }, |
| { |
| "epoch": 2.9, |
| "learning_rate": 1.1161846778285133e-08, |
| "logits/chosen": -2.2622501850128174, |
| "logits/rejected": -1.8619012832641602, |
| "logps/chosen": -87.53951263427734, |
| "logps/rejected": -131.9397430419922, |
| "loss": 0.0025, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.8194286227226257, |
| "rewards/margins": 32.57026290893555, |
| "rewards/rejected": -31.750835418701172, |
| "step": 6350 |
| }, |
| { |
| "epoch": 2.9, |
| "learning_rate": 1.06544901065449e-08, |
| "logits/chosen": -2.26688814163208, |
| "logits/rejected": -1.9139961004257202, |
| "logps/chosen": -85.9757308959961, |
| "logps/rejected": -134.81204223632812, |
| "loss": 0.0031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7707549333572388, |
| "rewards/margins": 33.59237289428711, |
| "rewards/rejected": -32.821617126464844, |
| "step": 6360 |
| }, |
| { |
| "epoch": 2.91, |
| "learning_rate": 1.0147133434804666e-08, |
| "logits/chosen": -2.1150248050689697, |
| "logits/rejected": -1.7503303289413452, |
| "logps/chosen": -88.97874450683594, |
| "logps/rejected": -134.1490478515625, |
| "loss": 0.0085, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2682693004608154, |
| "rewards/margins": 32.085575103759766, |
| "rewards/rejected": -30.817302703857422, |
| "step": 6370 |
| }, |
| { |
| "epoch": 2.91, |
| "learning_rate": 9.639776763064434e-09, |
| "logits/chosen": -2.1985535621643066, |
| "logits/rejected": -1.8416798114776611, |
| "logps/chosen": -83.56170654296875, |
| "logps/rejected": -127.2623291015625, |
| "loss": 0.0052, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5547320246696472, |
| "rewards/margins": 31.148799896240234, |
| "rewards/rejected": -30.594066619873047, |
| "step": 6380 |
| }, |
| { |
| "epoch": 2.92, |
| "learning_rate": 9.1324200913242e-09, |
| "logits/chosen": -2.1735215187072754, |
| "logits/rejected": -1.806305170059204, |
| "logps/chosen": -85.32138061523438, |
| "logps/rejected": -126.041015625, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6665178537368774, |
| "rewards/margins": 31.99408531188965, |
| "rewards/rejected": -30.327566146850586, |
| "step": 6390 |
| }, |
| { |
| "epoch": 2.92, |
| "learning_rate": 8.625063419583967e-09, |
| "logits/chosen": -2.2531039714813232, |
| "logits/rejected": -1.8883110284805298, |
| "logps/chosen": -89.10762023925781, |
| "logps/rejected": -135.33935546875, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.8338155746459961, |
| "rewards/margins": 32.7072868347168, |
| "rewards/rejected": -31.873470306396484, |
| "step": 6400 |
| }, |
| { |
| "epoch": 2.92, |
| "eval_logits/chosen": -2.227215051651001, |
| "eval_logits/rejected": -1.8791638612747192, |
| "eval_logps/chosen": -87.51187133789062, |
| "eval_logps/rejected": -129.8273468017578, |
| "eval_loss": 0.005407842341810465, |
| "eval_rewards/accuracies": 0.994413435459137, |
| "eval_rewards/chosen": 0.6593578457832336, |
| "eval_rewards/margins": 32.15756607055664, |
| "eval_rewards/rejected": -31.498210906982422, |
| "eval_runtime": 185.4523, |
| "eval_samples_per_second": 15.433, |
| "eval_steps_per_second": 0.965, |
| "step": 6400 |
| }, |
| { |
| "epoch": 2.93, |
| "learning_rate": 8.117706747843734e-09, |
| "logits/chosen": -2.1412510871887207, |
| "logits/rejected": -1.8283464908599854, |
| "logps/chosen": -90.0947265625, |
| "logps/rejected": -132.66854858398438, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1616397649049759, |
| "rewards/margins": 31.50514793395996, |
| "rewards/rejected": -31.3435115814209, |
| "step": 6410 |
| }, |
| { |
| "epoch": 2.93, |
| "learning_rate": 7.6103500761035e-09, |
| "logits/chosen": -2.265721321105957, |
| "logits/rejected": -1.9056390523910522, |
| "logps/chosen": -87.0054931640625, |
| "logps/rejected": -132.97547912597656, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 1.0425851345062256, |
| "rewards/margins": 32.99910354614258, |
| "rewards/rejected": -31.95652198791504, |
| "step": 6420 |
| }, |
| { |
| "epoch": 2.94, |
| "learning_rate": 7.1029934043632664e-09, |
| "logits/chosen": -2.2558608055114746, |
| "logits/rejected": -1.8824846744537354, |
| "logps/chosen": -86.92906188964844, |
| "logps/rejected": -135.2639617919922, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9257342219352722, |
| "rewards/margins": 33.13534927368164, |
| "rewards/rejected": -32.209617614746094, |
| "step": 6430 |
| }, |
| { |
| "epoch": 2.94, |
| "learning_rate": 6.595636732623033e-09, |
| "logits/chosen": -2.2064738273620605, |
| "logits/rejected": -1.859531044960022, |
| "logps/chosen": -87.9058837890625, |
| "logps/rejected": -133.92010498046875, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3929450511932373, |
| "rewards/margins": 33.6112174987793, |
| "rewards/rejected": -32.2182731628418, |
| "step": 6440 |
| }, |
| { |
| "epoch": 2.94, |
| "learning_rate": 6.0882800608828e-09, |
| "logits/chosen": -2.2518255710601807, |
| "logits/rejected": -1.9297151565551758, |
| "logps/chosen": -82.05953979492188, |
| "logps/rejected": -129.396240234375, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9110031127929688, |
| "rewards/margins": 31.361520767211914, |
| "rewards/rejected": -30.450519561767578, |
| "step": 6450 |
| }, |
| { |
| "epoch": 2.95, |
| "learning_rate": 5.580923389142566e-09, |
| "logits/chosen": -2.0963399410247803, |
| "logits/rejected": -1.7235018014907837, |
| "logps/chosen": -89.75813293457031, |
| "logps/rejected": -132.9015350341797, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 2.336916923522949, |
| "rewards/margins": 32.56396484375, |
| "rewards/rejected": -30.2270450592041, |
| "step": 6460 |
| }, |
| { |
| "epoch": 2.95, |
| "learning_rate": 5.073566717402333e-09, |
| "logits/chosen": -2.23740816116333, |
| "logits/rejected": -1.8160254955291748, |
| "logps/chosen": -97.89299774169922, |
| "logps/rejected": -127.76078796386719, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.258462905883789, |
| "rewards/margins": 32.506343841552734, |
| "rewards/rejected": -31.247879028320312, |
| "step": 6470 |
| }, |
| { |
| "epoch": 2.96, |
| "learning_rate": 4.5662100456621e-09, |
| "logits/chosen": -2.185804843902588, |
| "logits/rejected": -1.8170665502548218, |
| "logps/chosen": -86.79978942871094, |
| "logps/rejected": -129.39993286132812, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.790616989135742, |
| "rewards/margins": 33.8178825378418, |
| "rewards/rejected": -31.027271270751953, |
| "step": 6480 |
| }, |
| { |
| "epoch": 2.96, |
| "learning_rate": 4.058853373921867e-09, |
| "logits/chosen": -2.232938766479492, |
| "logits/rejected": -1.811173439025879, |
| "logps/chosen": -90.50852966308594, |
| "logps/rejected": -131.22023010253906, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5173299312591553, |
| "rewards/margins": 34.080116271972656, |
| "rewards/rejected": -32.562782287597656, |
| "step": 6490 |
| }, |
| { |
| "epoch": 2.97, |
| "learning_rate": 3.5514967021816332e-09, |
| "logits/chosen": -2.2670958042144775, |
| "logits/rejected": -1.914764165878296, |
| "logps/chosen": -91.02118682861328, |
| "logps/rejected": -133.01803588867188, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9153534173965454, |
| "rewards/margins": 31.895156860351562, |
| "rewards/rejected": -30.979806900024414, |
| "step": 6500 |
| }, |
| { |
| "epoch": 2.97, |
| "eval_logits/chosen": -2.227478265762329, |
| "eval_logits/rejected": -1.8803960084915161, |
| "eval_logps/chosen": -87.42868041992188, |
| "eval_logps/rejected": -129.57211303710938, |
| "eval_loss": 0.005446174647659063, |
| "eval_rewards/accuracies": 0.9916201233863831, |
| "eval_rewards/chosen": 0.7009533643722534, |
| "eval_rewards/margins": 32.07155227661133, |
| "eval_rewards/rejected": -31.37059211730957, |
| "eval_runtime": 193.9275, |
| "eval_samples_per_second": 14.758, |
| "eval_steps_per_second": 0.923, |
| "step": 6500 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 6570, |
| "num_train_epochs": 3, |
| "save_steps": 500, |
| "total_flos": 0.0, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
